code_generator_x86.cc revision 6de1938e562b0d06e462512dd806166e754035ea 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "art_method.h"
20#include "code_generator_utils.h"
21#include "compiled_method.h"
22#include "entrypoints/quick/quick_entrypoints.h"
23#include "entrypoints/quick/quick_entrypoints_enum.h"
24#include "gc/accounting/card_table.h"
25#include "intrinsics.h"
26#include "intrinsics_x86.h"
27#include "mirror/array-inl.h"
28#include "mirror/class-inl.h"
29#include "pc_relative_fixups_x86.h"
30#include "thread.h"
31#include "utils/assembler.h"
32#include "utils/stack_checks.h"
33#include "utils/x86/assembler_x86.h"
34#include "utils/x86/managed_register_x86.h"
35
36namespace art {
37
38template<class MirrorType>
39class GcRoot;
40
41namespace x86 {
42
43static constexpr int kCurrentMethodStackOffset = 0;
44static constexpr Register kMethodRegisterArgument = EAX;
45static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI };
46
47static constexpr int kC2ConditionMask = 0x400;
48
49static constexpr int kFakeReturnRegister = Register(8);
50
51#define __ down_cast<X86Assembler*>(codegen->GetAssembler())->
52#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kX86WordSize, x).Int32Value()
53
54class NullCheckSlowPathX86 : public SlowPathCode {
55 public:
56  explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {}
57
58  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
59    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
60    __ Bind(GetEntryLabel());
61    if (instruction_->CanThrowIntoCatchBlock()) {
62      // Live registers will be restored in the catch block if caught.
63      SaveLiveRegisters(codegen, instruction_->GetLocations());
64    }
65    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowNullPointer),
66                               instruction_,
67                               instruction_->GetDexPc(),
68                               this);
69    CheckEntrypointTypes<kQuickThrowNullPointer, void, void>();
70  }
71
72  bool IsFatal() const OVERRIDE { return true; }
73
74  const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathX86"; }
75
76 private:
77  HNullCheck* const instruction_;
78  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
79};
80
81class DivZeroCheckSlowPathX86 : public SlowPathCode {
82 public:
83  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {}
84
85  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
86    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
87    __ Bind(GetEntryLabel());
88    if (instruction_->CanThrowIntoCatchBlock()) {
89      // Live registers will be restored in the catch block if caught.
90      SaveLiveRegisters(codegen, instruction_->GetLocations());
91    }
92    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowDivZero),
93                               instruction_,
94                               instruction_->GetDexPc(),
95                               this);
96    CheckEntrypointTypes<kQuickThrowDivZero, void, void>();
97  }
98
99  bool IsFatal() const OVERRIDE { return true; }
100
101  const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathX86"; }
102
103 private:
104  HDivZeroCheck* const instruction_;
105  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
106};
107
108class DivRemMinusOneSlowPathX86 : public SlowPathCode {
109 public:
110  DivRemMinusOneSlowPathX86(Register reg, bool is_div) : reg_(reg), is_div_(is_div) {}
111
112  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
113    __ Bind(GetEntryLabel());
114    if (is_div_) {
115      __ negl(reg_);
116    } else {
117      __ movl(reg_, Immediate(0));
118    }
119    __ jmp(GetExitLabel());
120  }
121
122  const char* GetDescription() const OVERRIDE { return "DivRemMinusOneSlowPathX86"; }
123
124 private:
125  Register reg_;
126  bool is_div_;
127  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
128};
129
130class BoundsCheckSlowPathX86 : public SlowPathCode {
131 public:
132  explicit BoundsCheckSlowPathX86(HBoundsCheck* instruction) : instruction_(instruction) {}
133
134  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
135    LocationSummary* locations = instruction_->GetLocations();
136    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
137    __ Bind(GetEntryLabel());
138    // We're moving two locations to locations that could overlap, so we need a parallel
139    // move resolver.
140    if (instruction_->CanThrowIntoCatchBlock()) {
141      // Live registers will be restored in the catch block if caught.
142      SaveLiveRegisters(codegen, instruction_->GetLocations());
143    }
144    InvokeRuntimeCallingConvention calling_convention;
145    x86_codegen->EmitParallelMoves(
146        locations->InAt(0),
147        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
148        Primitive::kPrimInt,
149        locations->InAt(1),
150        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
151        Primitive::kPrimInt);
152    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowArrayBounds),
153                               instruction_,
154                               instruction_->GetDexPc(),
155                               this);
156    CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>();
157  }
158
159  bool IsFatal() const OVERRIDE { return true; }
160
161  const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathX86"; }
162
163 private:
164  HBoundsCheck* const instruction_;
165
166  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
167};
168
169class SuspendCheckSlowPathX86 : public SlowPathCode {
170 public:
171  SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
172      : instruction_(instruction), successor_(successor) {}
173
174  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
175    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
176    __ Bind(GetEntryLabel());
177    SaveLiveRegisters(codegen, instruction_->GetLocations());
178    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pTestSuspend),
179                               instruction_,
180                               instruction_->GetDexPc(),
181                               this);
182    CheckEntrypointTypes<kQuickTestSuspend, void, void>();
183    RestoreLiveRegisters(codegen, instruction_->GetLocations());
184    if (successor_ == nullptr) {
185      __ jmp(GetReturnLabel());
186    } else {
187      __ jmp(x86_codegen->GetLabelOf(successor_));
188    }
189  }
190
191  Label* GetReturnLabel() {
192    DCHECK(successor_ == nullptr);
193    return &return_label_;
194  }
195
196  HBasicBlock* GetSuccessor() const {
197    return successor_;
198  }
199
200  const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathX86"; }
201
202 private:
203  HSuspendCheck* const instruction_;
204  HBasicBlock* const successor_;
205  Label return_label_;
206
207  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
208};
209
210class LoadStringSlowPathX86 : public SlowPathCode {
211 public:
212  explicit LoadStringSlowPathX86(HLoadString* instruction) : instruction_(instruction) {}
213
214  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
215    LocationSummary* locations = instruction_->GetLocations();
216    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
217
218    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
219    __ Bind(GetEntryLabel());
220    SaveLiveRegisters(codegen, locations);
221
222    InvokeRuntimeCallingConvention calling_convention;
223    __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex()));
224    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pResolveString),
225                               instruction_,
226                               instruction_->GetDexPc(),
227                               this);
228    CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
229    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
230    RestoreLiveRegisters(codegen, locations);
231
232    __ jmp(GetExitLabel());
233  }
234
235  const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathX86"; }
236
237 private:
238  HLoadString* const instruction_;
239
240  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
241};
242
243class LoadClassSlowPathX86 : public SlowPathCode {
244 public:
245  LoadClassSlowPathX86(HLoadClass* cls,
246                       HInstruction* at,
247                       uint32_t dex_pc,
248                       bool do_clinit)
249      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
250    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
251  }
252
253  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
254    LocationSummary* locations = at_->GetLocations();
255    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
256    __ Bind(GetEntryLabel());
257    SaveLiveRegisters(codegen, locations);
258
259    InvokeRuntimeCallingConvention calling_convention;
260    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
261    x86_codegen->InvokeRuntime(do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
262                                          : QUICK_ENTRY_POINT(pInitializeType),
263                               at_, dex_pc_, this);
264    if (do_clinit_) {
265      CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>();
266    } else {
267      CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>();
268    }
269
270    // Move the class to the desired location.
271    Location out = locations->Out();
272    if (out.IsValid()) {
273      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
274      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
275    }
276
277    RestoreLiveRegisters(codegen, locations);
278    __ jmp(GetExitLabel());
279  }
280
281  const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathX86"; }
282
283 private:
284  // The class this slow path will load.
285  HLoadClass* const cls_;
286
287  // The instruction where this slow path is happening.
288  // (Might be the load class or an initialization check).
289  HInstruction* const at_;
290
291  // The dex PC of `at_`.
292  const uint32_t dex_pc_;
293
294  // Whether to initialize the class.
295  const bool do_clinit_;
296
297  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
298};
299
300class TypeCheckSlowPathX86 : public SlowPathCode {
301 public:
302  TypeCheckSlowPathX86(HInstruction* instruction, bool is_fatal)
303      : instruction_(instruction), is_fatal_(is_fatal) {}
304
305  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
306    LocationSummary* locations = instruction_->GetLocations();
307    Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0)
308                                                        : locations->Out();
309    DCHECK(instruction_->IsCheckCast()
310           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
311
312    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
313    __ Bind(GetEntryLabel());
314
315    if (!is_fatal_) {
316      SaveLiveRegisters(codegen, locations);
317    }
318
319    // We're moving two locations to locations that could overlap, so we need a parallel
320    // move resolver.
321    InvokeRuntimeCallingConvention calling_convention;
322    x86_codegen->EmitParallelMoves(
323        locations->InAt(1),
324        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
325        Primitive::kPrimNot,
326        object_class,
327        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
328        Primitive::kPrimNot);
329
330    if (instruction_->IsInstanceOf()) {
331      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
332                                 instruction_,
333                                 instruction_->GetDexPc(),
334                                 this);
335      CheckEntrypointTypes<
336          kQuickInstanceofNonTrivial, uint32_t, const mirror::Class*, const mirror::Class*>();
337    } else {
338      DCHECK(instruction_->IsCheckCast());
339      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
340                                 instruction_,
341                                 instruction_->GetDexPc(),
342                                 this);
343      CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
344    }
345
346    if (!is_fatal_) {
347      if (instruction_->IsInstanceOf()) {
348        x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
349      }
350      RestoreLiveRegisters(codegen, locations);
351
352      __ jmp(GetExitLabel());
353    }
354  }
355
356  const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathX86"; }
357  bool IsFatal() const OVERRIDE { return is_fatal_; }
358
359 private:
360  HInstruction* const instruction_;
361  const bool is_fatal_;
362
363  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
364};
365
366class DeoptimizationSlowPathX86 : public SlowPathCode {
367 public:
368  explicit DeoptimizationSlowPathX86(HDeoptimize* instruction)
369    : instruction_(instruction) {}
370
371  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
372    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
373    __ Bind(GetEntryLabel());
374    SaveLiveRegisters(codegen, instruction_->GetLocations());
375    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize),
376                               instruction_,
377                               instruction_->GetDexPc(),
378                               this);
379    CheckEntrypointTypes<kQuickDeoptimize, void, void>();
380  }
381
382  const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathX86"; }
383
384 private:
385  HDeoptimize* const instruction_;
386  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86);
387};
388
389class ArraySetSlowPathX86 : public SlowPathCode {
390 public:
391  explicit ArraySetSlowPathX86(HInstruction* instruction) : instruction_(instruction) {}
392
393  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
394    LocationSummary* locations = instruction_->GetLocations();
395    __ Bind(GetEntryLabel());
396    SaveLiveRegisters(codegen, locations);
397
398    InvokeRuntimeCallingConvention calling_convention;
399    HParallelMove parallel_move(codegen->GetGraph()->GetArena());
400    parallel_move.AddMove(
401        locations->InAt(0),
402        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
403        Primitive::kPrimNot,
404        nullptr);
405    parallel_move.AddMove(
406        locations->InAt(1),
407        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
408        Primitive::kPrimInt,
409        nullptr);
410    parallel_move.AddMove(
411        locations->InAt(2),
412        Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
413        Primitive::kPrimNot,
414        nullptr);
415    codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
416
417    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
418    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
419                               instruction_,
420                               instruction_->GetDexPc(),
421                               this);
422    CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>();
423    RestoreLiveRegisters(codegen, locations);
424    __ jmp(GetExitLabel());
425  }
426
427  const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathX86"; }
428
429 private:
430  HInstruction* const instruction_;
431
432  DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathX86);
433};
434
435// Slow path marking an object during a read barrier.
436class ReadBarrierMarkSlowPathX86 : public SlowPathCode {
437 public:
438  ReadBarrierMarkSlowPathX86(HInstruction* instruction, Location out, Location obj)
439      : instruction_(instruction), out_(out), obj_(obj) {
440    DCHECK(kEmitCompilerReadBarrier);
441  }
442
443  const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathX86"; }
444
445  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
446    LocationSummary* locations = instruction_->GetLocations();
447    Register reg_out = out_.AsRegister<Register>();
448    DCHECK(locations->CanCall());
449    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
450    DCHECK(instruction_->IsInstanceFieldGet() ||
451           instruction_->IsStaticFieldGet() ||
452           instruction_->IsArrayGet() ||
453           instruction_->IsLoadClass() ||
454           instruction_->IsLoadString() ||
455           instruction_->IsInstanceOf() ||
456           instruction_->IsCheckCast())
457        << "Unexpected instruction in read barrier marking slow path: "
458        << instruction_->DebugName();
459
460    __ Bind(GetEntryLabel());
461    SaveLiveRegisters(codegen, locations);
462
463    InvokeRuntimeCallingConvention calling_convention;
464    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
465    x86_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), obj_);
466    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierMark),
467                               instruction_,
468                               instruction_->GetDexPc(),
469                               this);
470    CheckEntrypointTypes<kQuickReadBarrierMark, mirror::Object*, mirror::Object*>();
471    x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
472
473    RestoreLiveRegisters(codegen, locations);
474    __ jmp(GetExitLabel());
475  }
476
477 private:
478  HInstruction* const instruction_;
479  const Location out_;
480  const Location obj_;
481
482  DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathX86);
483};
484
485// Slow path generating a read barrier for a heap reference.
486class ReadBarrierForHeapReferenceSlowPathX86 : public SlowPathCode {
487 public:
488  ReadBarrierForHeapReferenceSlowPathX86(HInstruction* instruction,
489                                         Location out,
490                                         Location ref,
491                                         Location obj,
492                                         uint32_t offset,
493                                         Location index)
494      : instruction_(instruction),
495        out_(out),
496        ref_(ref),
497        obj_(obj),
498        offset_(offset),
499        index_(index) {
500    DCHECK(kEmitCompilerReadBarrier);
501    // If `obj` is equal to `out` or `ref`, it means the initial object
502    // has been overwritten by (or after) the heap object reference load
503    // to be instrumented, e.g.:
504    //
505    //   __ movl(out, Address(out, offset));
506    //   codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset);
507    //
508    // In that case, we have lost the information about the original
509    // object, and the emitted read barrier cannot work properly.
510    DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out;
511    DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref;
512  }
513
514  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
515    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
516    LocationSummary* locations = instruction_->GetLocations();
517    Register reg_out = out_.AsRegister<Register>();
518    DCHECK(locations->CanCall());
519    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
520    DCHECK(!instruction_->IsInvoke() ||
521           (instruction_->IsInvokeStaticOrDirect() &&
522            instruction_->GetLocations()->Intrinsified()))
523        << "Unexpected instruction in read barrier for heap reference slow path: "
524        << instruction_->DebugName();
525
526    __ Bind(GetEntryLabel());
527    SaveLiveRegisters(codegen, locations);
528
529    // We may have to change the index's value, but as `index_` is a
530    // constant member (like other "inputs" of this slow path),
531    // introduce a copy of it, `index`.
532    Location index = index_;
533    if (index_.IsValid()) {
534      // Handle `index_` for HArrayGet and intrinsic UnsafeGetObject.
535      if (instruction_->IsArrayGet()) {
536        // Compute the actual memory offset and store it in `index`.
537        Register index_reg = index_.AsRegister<Register>();
538        DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg));
539        if (codegen->IsCoreCalleeSaveRegister(index_reg)) {
540          // We are about to change the value of `index_reg` (see the
541          // calls to art::x86::X86Assembler::shll and
542          // art::x86::X86Assembler::AddImmediate below), but it has
543          // not been saved by the previous call to
544          // art::SlowPathCode::SaveLiveRegisters, as it is a
545          // callee-save register --
546          // art::SlowPathCode::SaveLiveRegisters does not consider
547          // callee-save registers, as it has been designed with the
548          // assumption that callee-save registers are supposed to be
549          // handled by the called function.  So, as a callee-save
550          // register, `index_reg` _would_ eventually be saved onto
551          // the stack, but it would be too late: we would have
552          // changed its value earlier.  Therefore, we manually save
553          // it here into another freely available register,
554          // `free_reg`, chosen of course among the caller-save
555          // registers (as a callee-save `free_reg` register would
556          // exhibit the same problem).
557          //
558          // Note we could have requested a temporary register from
559          // the register allocator instead; but we prefer not to, as
560          // this is a slow path, and we know we can find a
561          // caller-save register that is available.
562          Register free_reg = FindAvailableCallerSaveRegister(codegen);
563          __ movl(free_reg, index_reg);
564          index_reg = free_reg;
565          index = Location::RegisterLocation(index_reg);
566        } else {
567          // The initial register stored in `index_` has already been
568          // saved in the call to art::SlowPathCode::SaveLiveRegisters
569          // (as it is not a callee-save register), so we can freely
570          // use it.
571        }
572        // Shifting the index value contained in `index_reg` by the scale
573        // factor (2) cannot overflow in practice, as the runtime is
574        // unable to allocate object arrays with a size larger than
575        // 2^26 - 1 (that is, 2^28 - 4 bytes).
576        __ shll(index_reg, Immediate(TIMES_4));
577        static_assert(
578            sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
579            "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
580        __ AddImmediate(index_reg, Immediate(offset_));
581      } else {
582        DCHECK(instruction_->IsInvoke());
583        DCHECK(instruction_->GetLocations()->Intrinsified());
584        DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) ||
585               (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile))
586            << instruction_->AsInvoke()->GetIntrinsic();
587        DCHECK_EQ(offset_, 0U);
588        DCHECK(index_.IsRegisterPair());
589        // UnsafeGet's offset location is a register pair, the low
590        // part contains the correct offset.
591        index = index_.ToLow();
592      }
593    }
594
595    // We're moving two or three locations to locations that could
596    // overlap, so we need a parallel move resolver.
597    InvokeRuntimeCallingConvention calling_convention;
598    HParallelMove parallel_move(codegen->GetGraph()->GetArena());
599    parallel_move.AddMove(ref_,
600                          Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
601                          Primitive::kPrimNot,
602                          nullptr);
603    parallel_move.AddMove(obj_,
604                          Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
605                          Primitive::kPrimNot,
606                          nullptr);
607    if (index.IsValid()) {
608      parallel_move.AddMove(index,
609                            Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
610                            Primitive::kPrimInt,
611                            nullptr);
612      codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
613    } else {
614      codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
615      __ movl(calling_convention.GetRegisterAt(2), Immediate(offset_));
616    }
617    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierSlow),
618                               instruction_,
619                               instruction_->GetDexPc(),
620                               this);
621    CheckEntrypointTypes<
622        kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>();
623    x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
624
625    RestoreLiveRegisters(codegen, locations);
626    __ jmp(GetExitLabel());
627  }
628
629  const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathX86"; }
630
631 private:
632  Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) {
633    size_t ref = static_cast<int>(ref_.AsRegister<Register>());
634    size_t obj = static_cast<int>(obj_.AsRegister<Register>());
635    for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
636      if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) {
637        return static_cast<Register>(i);
638      }
639    }
640    // We shall never fail to find a free caller-save register, as
641    // there are more than two core caller-save registers on x86
642    // (meaning it is possible to find one which is different from
643    // `ref` and `obj`).
644    DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u);
645    LOG(FATAL) << "Could not find a free caller-save register";
646    UNREACHABLE();
647  }
648
649  HInstruction* const instruction_;
650  const Location out_;
651  const Location ref_;
652  const Location obj_;
653  const uint32_t offset_;
654  // An additional location containing an index to an array.
655  // Only used for HArrayGet and the UnsafeGetObject &
656  // UnsafeGetObjectVolatile intrinsics.
657  const Location index_;
658
659  DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathX86);
660};
661
662// Slow path generating a read barrier for a GC root.
663class ReadBarrierForRootSlowPathX86 : public SlowPathCode {
664 public:
665  ReadBarrierForRootSlowPathX86(HInstruction* instruction, Location out, Location root)
666      : instruction_(instruction), out_(out), root_(root) {
667    DCHECK(kEmitCompilerReadBarrier);
668  }
669
670  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
671    LocationSummary* locations = instruction_->GetLocations();
672    Register reg_out = out_.AsRegister<Register>();
673    DCHECK(locations->CanCall());
674    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
675    DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString())
676        << "Unexpected instruction in read barrier for GC root slow path: "
677        << instruction_->DebugName();
678
679    __ Bind(GetEntryLabel());
680    SaveLiveRegisters(codegen, locations);
681
682    InvokeRuntimeCallingConvention calling_convention;
683    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
684    x86_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_);
685    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierForRootSlow),
686                               instruction_,
687                               instruction_->GetDexPc(),
688                               this);
689    CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>();
690    x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
691
692    RestoreLiveRegisters(codegen, locations);
693    __ jmp(GetExitLabel());
694  }
695
696  const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathX86"; }
697
698 private:
699  HInstruction* const instruction_;
700  const Location out_;
701  const Location root_;
702
703  DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathX86);
704};
705
706#undef __
707#define __ down_cast<X86Assembler*>(GetAssembler())->
708
709inline Condition X86Condition(IfCondition cond) {
710  switch (cond) {
711    case kCondEQ: return kEqual;
712    case kCondNE: return kNotEqual;
713    case kCondLT: return kLess;
714    case kCondLE: return kLessEqual;
715    case kCondGT: return kGreater;
716    case kCondGE: return kGreaterEqual;
717    case kCondB:  return kBelow;
718    case kCondBE: return kBelowEqual;
719    case kCondA:  return kAbove;
720    case kCondAE: return kAboveEqual;
721  }
722  LOG(FATAL) << "Unreachable";
723  UNREACHABLE();
724}
725
726// Maps signed condition to unsigned condition and FP condition to x86 name.
727inline Condition X86UnsignedOrFPCondition(IfCondition cond) {
728  switch (cond) {
729    case kCondEQ: return kEqual;
730    case kCondNE: return kNotEqual;
731    // Signed to unsigned, and FP to x86 name.
732    case kCondLT: return kBelow;
733    case kCondLE: return kBelowEqual;
734    case kCondGT: return kAbove;
735    case kCondGE: return kAboveEqual;
736    // Unsigned remain unchanged.
737    case kCondB:  return kBelow;
738    case kCondBE: return kBelowEqual;
739    case kCondA:  return kAbove;
740    case kCondAE: return kAboveEqual;
741  }
742  LOG(FATAL) << "Unreachable";
743  UNREACHABLE();
744}
745
746void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
747  stream << Register(reg);
748}
749
750void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
751  stream << XmmRegister(reg);
752}
753
754size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
755  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
756  return kX86WordSize;
757}
758
759size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
760  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
761  return kX86WordSize;
762}
763
764size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
765  __ movsd(Address(ESP, stack_index), XmmRegister(reg_id));
766  return GetFloatingPointSpillSlotSize();
767}
768
769size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
770  __ movsd(XmmRegister(reg_id), Address(ESP, stack_index));
771  return GetFloatingPointSpillSlotSize();
772}
773
774void CodeGeneratorX86::InvokeRuntime(QuickEntrypointEnum entrypoint,
775                                     HInstruction* instruction,
776                                     uint32_t dex_pc,
777                                     SlowPathCode* slow_path) {
778  InvokeRuntime(GetThreadOffset<kX86WordSize>(entrypoint).Int32Value(),
779                instruction,
780                dex_pc,
781                slow_path);
782}
783
784void CodeGeneratorX86::InvokeRuntime(int32_t entry_point_offset,
785                                     HInstruction* instruction,
786                                     uint32_t dex_pc,
787                                     SlowPathCode* slow_path) {
788  ValidateInvokeRuntime(instruction, slow_path);
789  __ fs()->call(Address::Absolute(entry_point_offset));
790  RecordPcInfo(instruction, dex_pc, slow_path);
791}
792
793CodeGeneratorX86::CodeGeneratorX86(HGraph* graph,
794                                   const X86InstructionSetFeatures& isa_features,
795                                   const CompilerOptions& compiler_options,
796                                   OptimizingCompilerStats* stats)
797    : CodeGenerator(graph,
798                    kNumberOfCpuRegisters,
799                    kNumberOfXmmRegisters,
800                    kNumberOfRegisterPairs,
801                    ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
802                                        arraysize(kCoreCalleeSaves))
803                        | (1 << kFakeReturnRegister),
804                    0,
805                    compiler_options,
806                    stats),
807      block_labels_(nullptr),
808      location_builder_(graph, this),
809      instruction_visitor_(graph, this),
810      move_resolver_(graph->GetArena(), this),
811      isa_features_(isa_features),
812      method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
813      relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
814      pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
815      fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
816  // Use a fake return address register to mimic Quick.
817  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
818}
819
820Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const {
821  switch (type) {
822    case Primitive::kPrimLong: {
823      size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs);
824      X86ManagedRegister pair =
825          X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg));
826      DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]);
827      DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]);
828      blocked_core_registers_[pair.AsRegisterPairLow()] = true;
829      blocked_core_registers_[pair.AsRegisterPairHigh()] = true;
830      UpdateBlockedPairRegisters();
831      return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
832    }
833
834    case Primitive::kPrimByte:
835    case Primitive::kPrimBoolean:
836    case Primitive::kPrimChar:
837    case Primitive::kPrimShort:
838    case Primitive::kPrimInt:
839    case Primitive::kPrimNot: {
840      Register reg = static_cast<Register>(
841          FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters));
842      // Block all register pairs that contain `reg`.
843      for (int i = 0; i < kNumberOfRegisterPairs; i++) {
844        X86ManagedRegister current =
845            X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
846        if (current.AsRegisterPairLow() == reg || current.AsRegisterPairHigh() == reg) {
847          blocked_register_pairs_[i] = true;
848        }
849      }
850      return Location::RegisterLocation(reg);
851    }
852
853    case Primitive::kPrimFloat:
854    case Primitive::kPrimDouble: {
855      return Location::FpuRegisterLocation(
856          FindFreeEntry(blocked_fpu_registers_, kNumberOfXmmRegisters));
857    }
858
859    case Primitive::kPrimVoid:
860      LOG(FATAL) << "Unreachable type " << type;
861  }
862
863  return Location::NoLocation();
864}
865
866void CodeGeneratorX86::SetupBlockedRegisters(bool is_baseline) const {
867  // Don't allocate the dalvik style register pair passing.
868  blocked_register_pairs_[ECX_EDX] = true;
869
870  // Stack register is always reserved.
871  blocked_core_registers_[ESP] = true;
872
873  if (is_baseline) {
874    blocked_core_registers_[EBP] = true;
875    blocked_core_registers_[ESI] = true;
876    blocked_core_registers_[EDI] = true;
877  }
878
879  UpdateBlockedPairRegisters();
880}
881
882void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
883  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
884    X86ManagedRegister current =
885        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
886    if (blocked_core_registers_[current.AsRegisterPairLow()]
887        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
888      blocked_register_pairs_[i] = true;
889    }
890  }
891}
892
893InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
894      : InstructionCodeGenerator(graph, codegen),
895        assembler_(codegen->GetAssembler()),
896        codegen_(codegen) {}
897
898static dwarf::Reg DWARFReg(Register reg) {
899  return dwarf::Reg::X86Core(static_cast<int>(reg));
900}
901
902void CodeGeneratorX86::GenerateFrameEntry() {
903  __ cfi().SetCurrentCFAOffset(kX86WordSize);  // return address
904  __ Bind(&frame_entry_label_);
905  bool skip_overflow_check =
906      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
907  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
908
909  if (!skip_overflow_check) {
910    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
911    RecordPcInfo(nullptr, 0);
912  }
913
914  if (HasEmptyFrame()) {
915    return;
916  }
917
918  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
919    Register reg = kCoreCalleeSaves[i];
920    if (allocated_registers_.ContainsCoreRegister(reg)) {
921      __ pushl(reg);
922      __ cfi().AdjustCFAOffset(kX86WordSize);
923      __ cfi().RelOffset(DWARFReg(reg), 0);
924    }
925  }
926
927  int adjust = GetFrameSize() - FrameEntrySpillSize();
928  __ subl(ESP, Immediate(adjust));
929  __ cfi().AdjustCFAOffset(adjust);
930  __ movl(Address(ESP, kCurrentMethodStackOffset), kMethodRegisterArgument);
931}
932
933void CodeGeneratorX86::GenerateFrameExit() {
934  __ cfi().RememberState();
935  if (!HasEmptyFrame()) {
936    int adjust = GetFrameSize() - FrameEntrySpillSize();
937    __ addl(ESP, Immediate(adjust));
938    __ cfi().AdjustCFAOffset(-adjust);
939
940    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
941      Register reg = kCoreCalleeSaves[i];
942      if (allocated_registers_.ContainsCoreRegister(reg)) {
943        __ popl(reg);
944        __ cfi().AdjustCFAOffset(-static_cast<int>(kX86WordSize));
945        __ cfi().Restore(DWARFReg(reg));
946      }
947    }
948  }
949  __ ret();
950  __ cfi().RestoreState();
951  __ cfi().DefCFAOffset(GetFrameSize());
952}
953
954void CodeGeneratorX86::Bind(HBasicBlock* block) {
955  __ Bind(GetLabelOf(block));
956}
957
958Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
959  switch (load->GetType()) {
960    case Primitive::kPrimLong:
961    case Primitive::kPrimDouble:
962      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
963
964    case Primitive::kPrimInt:
965    case Primitive::kPrimNot:
966    case Primitive::kPrimFloat:
967      return Location::StackSlot(GetStackSlot(load->GetLocal()));
968
969    case Primitive::kPrimBoolean:
970    case Primitive::kPrimByte:
971    case Primitive::kPrimChar:
972    case Primitive::kPrimShort:
973    case Primitive::kPrimVoid:
974      LOG(FATAL) << "Unexpected type " << load->GetType();
975      UNREACHABLE();
976  }
977
978  LOG(FATAL) << "Unreachable";
979  UNREACHABLE();
980}
981
982Location InvokeDexCallingConventionVisitorX86::GetReturnLocation(Primitive::Type type) const {
983  switch (type) {
984    case Primitive::kPrimBoolean:
985    case Primitive::kPrimByte:
986    case Primitive::kPrimChar:
987    case Primitive::kPrimShort:
988    case Primitive::kPrimInt:
989    case Primitive::kPrimNot:
990      return Location::RegisterLocation(EAX);
991
992    case Primitive::kPrimLong:
993      return Location::RegisterPairLocation(EAX, EDX);
994
995    case Primitive::kPrimVoid:
996      return Location::NoLocation();
997
998    case Primitive::kPrimDouble:
999    case Primitive::kPrimFloat:
1000      return Location::FpuRegisterLocation(XMM0);
1001  }
1002
1003  UNREACHABLE();
1004}
1005
1006Location InvokeDexCallingConventionVisitorX86::GetMethodLocation() const {
1007  return Location::RegisterLocation(kMethodRegisterArgument);
1008}
1009
1010Location InvokeDexCallingConventionVisitorX86::GetNextLocation(Primitive::Type type) {
1011  switch (type) {
1012    case Primitive::kPrimBoolean:
1013    case Primitive::kPrimByte:
1014    case Primitive::kPrimChar:
1015    case Primitive::kPrimShort:
1016    case Primitive::kPrimInt:
1017    case Primitive::kPrimNot: {
1018      uint32_t index = gp_index_++;
1019      stack_index_++;
1020      if (index < calling_convention.GetNumberOfRegisters()) {
1021        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
1022      } else {
1023        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
1024      }
1025    }
1026
1027    case Primitive::kPrimLong: {
1028      uint32_t index = gp_index_;
1029      gp_index_ += 2;
1030      stack_index_ += 2;
1031      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
1032        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
1033            calling_convention.GetRegisterPairAt(index));
1034        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
1035      } else {
1036        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
1037      }
1038    }
1039
1040    case Primitive::kPrimFloat: {
1041      uint32_t index = float_index_++;
1042      stack_index_++;
1043      if (index < calling_convention.GetNumberOfFpuRegisters()) {
1044        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
1045      } else {
1046        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
1047      }
1048    }
1049
1050    case Primitive::kPrimDouble: {
1051      uint32_t index = float_index_++;
1052      stack_index_ += 2;
1053      if (index < calling_convention.GetNumberOfFpuRegisters()) {
1054        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
1055      } else {
1056        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
1057      }
1058    }
1059
1060    case Primitive::kPrimVoid:
1061      LOG(FATAL) << "Unexpected parameter type " << type;
1062      break;
1063  }
1064  return Location::NoLocation();
1065}
1066
1067void CodeGeneratorX86::Move32(Location destination, Location source) {
1068  if (source.Equals(destination)) {
1069    return;
1070  }
1071  if (destination.IsRegister()) {
1072    if (source.IsRegister()) {
1073      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
1074    } else if (source.IsFpuRegister()) {
1075      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
1076    } else {
1077      DCHECK(source.IsStackSlot());
1078      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
1079    }
1080  } else if (destination.IsFpuRegister()) {
1081    if (source.IsRegister()) {
1082      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
1083    } else if (source.IsFpuRegister()) {
1084      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
1085    } else {
1086      DCHECK(source.IsStackSlot());
1087      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
1088    }
1089  } else {
1090    DCHECK(destination.IsStackSlot()) << destination;
1091    if (source.IsRegister()) {
1092      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
1093    } else if (source.IsFpuRegister()) {
1094      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
1095    } else if (source.IsConstant()) {
1096      HConstant* constant = source.GetConstant();
1097      int32_t value = GetInt32ValueOf(constant);
1098      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
1099    } else {
1100      DCHECK(source.IsStackSlot());
1101      __ pushl(Address(ESP, source.GetStackIndex()));
1102      __ popl(Address(ESP, destination.GetStackIndex()));
1103    }
1104  }
1105}
1106
1107void CodeGeneratorX86::Move64(Location destination, Location source) {
1108  if (source.Equals(destination)) {
1109    return;
1110  }
1111  if (destination.IsRegisterPair()) {
1112    if (source.IsRegisterPair()) {
1113      EmitParallelMoves(
1114          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
1115          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
1116          Primitive::kPrimInt,
1117          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
1118          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
1119          Primitive::kPrimInt);
1120    } else if (source.IsFpuRegister()) {
1121      XmmRegister src_reg = source.AsFpuRegister<XmmRegister>();
1122      __ movd(destination.AsRegisterPairLow<Register>(), src_reg);
1123      __ psrlq(src_reg, Immediate(32));
1124      __ movd(destination.AsRegisterPairHigh<Register>(), src_reg);
1125    } else {
1126      // No conflict possible, so just do the moves.
1127      DCHECK(source.IsDoubleStackSlot());
1128      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
1129      __ movl(destination.AsRegisterPairHigh<Register>(),
1130              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
1131    }
1132  } else if (destination.IsFpuRegister()) {
1133    if (source.IsFpuRegister()) {
1134      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
1135    } else if (source.IsDoubleStackSlot()) {
1136      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
1137    } else if (source.IsRegisterPair()) {
1138      size_t elem_size = Primitive::ComponentSize(Primitive::kPrimInt);
1139      // Create stack space for 2 elements.
1140      __ subl(ESP, Immediate(2 * elem_size));
1141      __ movl(Address(ESP, 0), source.AsRegisterPairLow<Register>());
1142      __ movl(Address(ESP, elem_size), source.AsRegisterPairHigh<Register>());
1143      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
1144      // And remove the temporary stack space we allocated.
1145      __ addl(ESP, Immediate(2 * elem_size));
1146    } else {
1147      LOG(FATAL) << "Unimplemented";
1148    }
1149  } else {
1150    DCHECK(destination.IsDoubleStackSlot()) << destination;
1151    if (source.IsRegisterPair()) {
1152      // No conflict possible, so just do the moves.
1153      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
1154      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
1155              source.AsRegisterPairHigh<Register>());
1156    } else if (source.IsFpuRegister()) {
1157      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
1158    } else if (source.IsConstant()) {
1159      HConstant* constant = source.GetConstant();
1160      int64_t value;
1161      if (constant->IsLongConstant()) {
1162        value = constant->AsLongConstant()->GetValue();
1163      } else {
1164        DCHECK(constant->IsDoubleConstant());
1165        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
1166      }
1167      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(Low32Bits(value)));
1168      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value)));
1169    } else {
1170      DCHECK(source.IsDoubleStackSlot()) << source;
1171      EmitParallelMoves(
1172          Location::StackSlot(source.GetStackIndex()),
1173          Location::StackSlot(destination.GetStackIndex()),
1174          Primitive::kPrimInt,
1175          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
1176          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)),
1177          Primitive::kPrimInt);
1178    }
1179  }
1180}
1181
1182void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstruction* move_for) {
1183  LocationSummary* locations = instruction->GetLocations();
1184  if (instruction->IsCurrentMethod()) {
1185    Move32(location, Location::StackSlot(kCurrentMethodStackOffset));
1186  } else if (locations != nullptr && locations->Out().Equals(location)) {
1187    return;
1188  } else if (locations != nullptr && locations->Out().IsConstant()) {
1189    HConstant* const_to_move = locations->Out().GetConstant();
1190    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
1191      Immediate imm(GetInt32ValueOf(const_to_move));
1192      if (location.IsRegister()) {
1193        __ movl(location.AsRegister<Register>(), imm);
1194      } else if (location.IsStackSlot()) {
1195        __ movl(Address(ESP, location.GetStackIndex()), imm);
1196      } else {
1197        DCHECK(location.IsConstant());
1198        DCHECK_EQ(location.GetConstant(), const_to_move);
1199      }
1200    } else if (const_to_move->IsLongConstant()) {
1201      int64_t value = const_to_move->AsLongConstant()->GetValue();
1202      if (location.IsRegisterPair()) {
1203        __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
1204        __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
1205      } else if (location.IsDoubleStackSlot()) {
1206        __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value)));
1207        __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)),
1208                Immediate(High32Bits(value)));
1209      } else {
1210        DCHECK(location.IsConstant());
1211        DCHECK_EQ(location.GetConstant(), instruction);
1212      }
1213    }
1214  } else if (instruction->IsTemporary()) {
1215    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
1216    if (temp_location.IsStackSlot()) {
1217      Move32(location, temp_location);
1218    } else {
1219      DCHECK(temp_location.IsDoubleStackSlot());
1220      Move64(location, temp_location);
1221    }
1222  } else if (instruction->IsLoadLocal()) {
1223    int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
1224    switch (instruction->GetType()) {
1225      case Primitive::kPrimBoolean:
1226      case Primitive::kPrimByte:
1227      case Primitive::kPrimChar:
1228      case Primitive::kPrimShort:
1229      case Primitive::kPrimInt:
1230      case Primitive::kPrimNot:
1231      case Primitive::kPrimFloat:
1232        Move32(location, Location::StackSlot(slot));
1233        break;
1234
1235      case Primitive::kPrimLong:
1236      case Primitive::kPrimDouble:
1237        Move64(location, Location::DoubleStackSlot(slot));
1238        break;
1239
1240      default:
1241        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
1242    }
1243  } else {
1244    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
1245    switch (instruction->GetType()) {
1246      case Primitive::kPrimBoolean:
1247      case Primitive::kPrimByte:
1248      case Primitive::kPrimChar:
1249      case Primitive::kPrimShort:
1250      case Primitive::kPrimInt:
1251      case Primitive::kPrimNot:
1252      case Primitive::kPrimFloat:
1253        Move32(location, locations->Out());
1254        break;
1255
1256      case Primitive::kPrimLong:
1257      case Primitive::kPrimDouble:
1258        Move64(location, locations->Out());
1259        break;
1260
1261      default:
1262        LOG(FATAL) << "Unexpected type " << instruction->GetType();
1263    }
1264  }
1265}
1266
1267void CodeGeneratorX86::MoveConstant(Location location, int32_t value) {
1268  DCHECK(location.IsRegister());
1269  __ movl(location.AsRegister<Register>(), Immediate(value));
1270}
1271
1272void CodeGeneratorX86::MoveLocation(Location dst, Location src, Primitive::Type dst_type) {
1273  if (Primitive::Is64BitType(dst_type)) {
1274    Move64(dst, src);
1275  } else {
1276    Move32(dst, src);
1277  }
1278}
1279
1280void CodeGeneratorX86::AddLocationAsTemp(Location location, LocationSummary* locations) {
1281  if (location.IsRegister()) {
1282    locations->AddTemp(location);
1283  } else if (location.IsRegisterPair()) {
1284    locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>()));
1285    locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>()));
1286  } else {
1287    UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location;
1288  }
1289}
1290
1291void InstructionCodeGeneratorX86::HandleGoto(HInstruction* got, HBasicBlock* successor) {
1292  DCHECK(!successor->IsExitBlock());
1293
1294  HBasicBlock* block = got->GetBlock();
1295  HInstruction* previous = got->GetPrevious();
1296
1297  HLoopInformation* info = block->GetLoopInformation();
1298  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
1299    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
1300    return;
1301  }
1302
1303  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
1304    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
1305  }
1306  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
1307    __ jmp(codegen_->GetLabelOf(successor));
1308  }
1309}
1310
1311void LocationsBuilderX86::VisitGoto(HGoto* got) {
1312  got->SetLocations(nullptr);
1313}
1314
1315void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
1316  HandleGoto(got, got->GetSuccessor());
1317}
1318
1319void LocationsBuilderX86::VisitTryBoundary(HTryBoundary* try_boundary) {
1320  try_boundary->SetLocations(nullptr);
1321}
1322
1323void InstructionCodeGeneratorX86::VisitTryBoundary(HTryBoundary* try_boundary) {
1324  HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
1325  if (!successor->IsExitBlock()) {
1326    HandleGoto(try_boundary, successor);
1327  }
1328}
1329
1330void LocationsBuilderX86::VisitExit(HExit* exit) {
1331  exit->SetLocations(nullptr);
1332}
1333
1334void InstructionCodeGeneratorX86::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
1335}
1336
1337template<class LabelType>
1338void InstructionCodeGeneratorX86::GenerateFPJumps(HCondition* cond,
1339                                                  LabelType* true_label,
1340                                                  LabelType* false_label) {
1341  if (cond->IsFPConditionTrueIfNaN()) {
1342    __ j(kUnordered, true_label);
1343  } else if (cond->IsFPConditionFalseIfNaN()) {
1344    __ j(kUnordered, false_label);
1345  }
1346  __ j(X86UnsignedOrFPCondition(cond->GetCondition()), true_label);
1347}
1348
1349template<class LabelType>
1350void InstructionCodeGeneratorX86::GenerateLongComparesAndJumps(HCondition* cond,
1351                                                               LabelType* true_label,
1352                                                               LabelType* false_label) {
1353  LocationSummary* locations = cond->GetLocations();
1354  Location left = locations->InAt(0);
1355  Location right = locations->InAt(1);
1356  IfCondition if_cond = cond->GetCondition();
1357
1358  Register left_high = left.AsRegisterPairHigh<Register>();
1359  Register left_low = left.AsRegisterPairLow<Register>();
1360  IfCondition true_high_cond = if_cond;
1361  IfCondition false_high_cond = cond->GetOppositeCondition();
1362  Condition final_condition = X86UnsignedOrFPCondition(if_cond);  // unsigned on lower part
1363
1364  // Set the conditions for the test, remembering that == needs to be
1365  // decided using the low words.
1366  switch (if_cond) {
1367    case kCondEQ:
1368    case kCondNE:
1369      // Nothing to do.
1370      break;
1371    case kCondLT:
1372      false_high_cond = kCondGT;
1373      break;
1374    case kCondLE:
1375      true_high_cond = kCondLT;
1376      break;
1377    case kCondGT:
1378      false_high_cond = kCondLT;
1379      break;
1380    case kCondGE:
1381      true_high_cond = kCondGT;
1382      break;
1383    case kCondB:
1384      false_high_cond = kCondA;
1385      break;
1386    case kCondBE:
1387      true_high_cond = kCondB;
1388      break;
1389    case kCondA:
1390      false_high_cond = kCondB;
1391      break;
1392    case kCondAE:
1393      true_high_cond = kCondA;
1394      break;
1395  }
1396
1397  if (right.IsConstant()) {
1398    int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
1399    int32_t val_high = High32Bits(value);
1400    int32_t val_low = Low32Bits(value);
1401
1402    if (val_high == 0) {
1403      __ testl(left_high, left_high);
1404    } else {
1405      __ cmpl(left_high, Immediate(val_high));
1406    }
1407    if (if_cond == kCondNE) {
1408      __ j(X86Condition(true_high_cond), true_label);
1409    } else if (if_cond == kCondEQ) {
1410      __ j(X86Condition(false_high_cond), false_label);
1411    } else {
1412      __ j(X86Condition(true_high_cond), true_label);
1413      __ j(X86Condition(false_high_cond), false_label);
1414    }
1415    // Must be equal high, so compare the lows.
1416    if (val_low == 0) {
1417      __ testl(left_low, left_low);
1418    } else {
1419      __ cmpl(left_low, Immediate(val_low));
1420    }
1421  } else {
1422    Register right_high = right.AsRegisterPairHigh<Register>();
1423    Register right_low = right.AsRegisterPairLow<Register>();
1424
1425    __ cmpl(left_high, right_high);
1426    if (if_cond == kCondNE) {
1427      __ j(X86Condition(true_high_cond), true_label);
1428    } else if (if_cond == kCondEQ) {
1429      __ j(X86Condition(false_high_cond), false_label);
1430    } else {
1431      __ j(X86Condition(true_high_cond), true_label);
1432      __ j(X86Condition(false_high_cond), false_label);
1433    }
1434    // Must be equal high, so compare the lows.
1435    __ cmpl(left_low, right_low);
1436  }
1437  // The last comparison might be unsigned.
1438  __ j(final_condition, true_label);
1439}
1440
1441template<class LabelType>
1442void InstructionCodeGeneratorX86::GenerateCompareTestAndBranch(HCondition* condition,
1443                                                               LabelType* true_target_in,
1444                                                               LabelType* false_target_in) {
1445  // Generated branching requires both targets to be explicit. If either of the
1446  // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead.
1447  LabelType fallthrough_target;
1448  LabelType* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in;
1449  LabelType* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in;
1450
1451  LocationSummary* locations = condition->GetLocations();
1452  Location left = locations->InAt(0);
1453  Location right = locations->InAt(1);
1454
1455  Primitive::Type type = condition->InputAt(0)->GetType();
1456  switch (type) {
1457    case Primitive::kPrimLong:
1458      GenerateLongComparesAndJumps(condition, true_target, false_target);
1459      break;
1460    case Primitive::kPrimFloat:
1461      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1462      GenerateFPJumps(condition, true_target, false_target);
1463      break;
1464    case Primitive::kPrimDouble:
1465      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1466      GenerateFPJumps(condition, true_target, false_target);
1467      break;
1468    default:
1469      LOG(FATAL) << "Unexpected compare type " << type;
1470  }
1471
1472  if (false_target != &fallthrough_target) {
1473    __ jmp(false_target);
1474  }
1475
1476  if (fallthrough_target.IsLinked()) {
1477    __ Bind(&fallthrough_target);
1478  }
1479}
1480
1481static bool AreEflagsSetFrom(HInstruction* cond, HInstruction* branch) {
1482  // Moves may affect the eflags register (move zero uses xorl), so the EFLAGS
1483  // are set only strictly before `branch`. We can't use the eflags on long/FP
1484  // conditions if they are materialized due to the complex branching.
1485  return cond->IsCondition() &&
1486         cond->GetNext() == branch &&
1487         cond->InputAt(0)->GetType() != Primitive::kPrimLong &&
1488         !Primitive::IsFloatingPointType(cond->InputAt(0)->GetType());
1489}
1490
1491template<class LabelType>
1492void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instruction,
1493                                                        size_t condition_input_index,
1494                                                        LabelType* true_target,
1495                                                        LabelType* false_target) {
1496  HInstruction* cond = instruction->InputAt(condition_input_index);
1497
1498  if (true_target == nullptr && false_target == nullptr) {
1499    // Nothing to do. The code always falls through.
1500    return;
1501  } else if (cond->IsIntConstant()) {
1502    // Constant condition, statically compared against 1.
1503    if (cond->AsIntConstant()->IsOne()) {
1504      if (true_target != nullptr) {
1505        __ jmp(true_target);
1506      }
1507    } else {
1508      DCHECK(cond->AsIntConstant()->IsZero());
1509      if (false_target != nullptr) {
1510        __ jmp(false_target);
1511      }
1512    }
1513    return;
1514  }
1515
1516  // The following code generates these patterns:
1517  //  (1) true_target == nullptr && false_target != nullptr
1518  //        - opposite condition true => branch to false_target
1519  //  (2) true_target != nullptr && false_target == nullptr
1520  //        - condition true => branch to true_target
1521  //  (3) true_target != nullptr && false_target != nullptr
1522  //        - condition true => branch to true_target
1523  //        - branch to false_target
1524  if (IsBooleanValueOrMaterializedCondition(cond)) {
1525    if (AreEflagsSetFrom(cond, instruction)) {
1526      if (true_target == nullptr) {
1527        __ j(X86Condition(cond->AsCondition()->GetOppositeCondition()), false_target);
1528      } else {
1529        __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
1530      }
1531    } else {
1532      // Materialized condition, compare against 0.
1533      Location lhs = instruction->GetLocations()->InAt(condition_input_index);
1534      if (lhs.IsRegister()) {
1535        __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1536      } else {
1537        __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
1538      }
1539      if (true_target == nullptr) {
1540        __ j(kEqual, false_target);
1541      } else {
1542        __ j(kNotEqual, true_target);
1543      }
1544    }
1545  } else {
1546    // Condition has not been materialized, use its inputs as the comparison and
1547    // its condition as the branch condition.
1548    HCondition* condition = cond->AsCondition();
1549
1550    // If this is a long or FP comparison that has been folded into
1551    // the HCondition, generate the comparison directly.
1552    Primitive::Type type = condition->InputAt(0)->GetType();
1553    if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
1554      GenerateCompareTestAndBranch(condition, true_target, false_target);
1555      return;
1556    }
1557
1558    Location lhs = condition->GetLocations()->InAt(0);
1559    Location rhs = condition->GetLocations()->InAt(1);
1560    // LHS is guaranteed to be in a register (see LocationsBuilderX86::HandleCondition).
1561    if (rhs.IsRegister()) {
1562      __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1563    } else if (rhs.IsConstant()) {
1564      int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1565      if (constant == 0) {
1566        __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1567      } else {
1568        __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1569      }
1570    } else {
1571      __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1572    }
1573    if (true_target == nullptr) {
1574      __ j(X86Condition(condition->GetOppositeCondition()), false_target);
1575    } else {
1576      __ j(X86Condition(condition->GetCondition()), true_target);
1577    }
1578  }
1579
1580  // If neither branch falls through (case 3), the conditional branch to `true_target`
1581  // was already emitted (case 2) and we need to emit a jump to `false_target`.
1582  if (true_target != nullptr && false_target != nullptr) {
1583    __ jmp(false_target);
1584  }
1585}
1586
1587void LocationsBuilderX86::VisitIf(HIf* if_instr) {
1588  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
1589  if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
1590    locations->SetInAt(0, Location::Any());
1591  }
1592}
1593
1594void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
1595  HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
1596  HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
1597  Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
1598      nullptr : codegen_->GetLabelOf(true_successor);
1599  Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
1600      nullptr : codegen_->GetLabelOf(false_successor);
1601  GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
1602}
1603
1604void LocationsBuilderX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1605  LocationSummary* locations = new (GetGraph()->GetArena())
1606      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
1607  if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
1608    locations->SetInAt(0, Location::Any());
1609  }
1610}
1611
1612void InstructionCodeGeneratorX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1613  SlowPathCode* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathX86>(deoptimize);
1614  GenerateTestAndBranch(deoptimize,
1615                        /* condition_input_index */ 0,
1616                        slow_path->GetEntryLabel(),
1617                        /* false_target */ static_cast<Label*>(nullptr));
1618}
1619
1620void LocationsBuilderX86::VisitNativeDebugInfo(HNativeDebugInfo* info) {
1621  new (GetGraph()->GetArena()) LocationSummary(info);
1622}
1623
1624void InstructionCodeGeneratorX86::VisitNativeDebugInfo(HNativeDebugInfo* info) {
1625  if (codegen_->HasStackMapAtCurrentPc()) {
1626    // Ensure that we do not collide with the stack map of the previous instruction.
1627    __ nop();
1628  }
1629  codegen_->RecordPcInfo(info, info->GetDexPc());
1630}
1631
1632void LocationsBuilderX86::VisitLocal(HLocal* local) {
1633  local->SetLocations(nullptr);
1634}
1635
1636void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
1637  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
1638}
1639
1640void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
1641  local->SetLocations(nullptr);
1642}
1643
1644void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_UNUSED) {
1645  // Nothing to do, this is driven by the code generator.
1646}
1647
1648void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
1649  LocationSummary* locations =
1650      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
1651  switch (store->InputAt(1)->GetType()) {
1652    case Primitive::kPrimBoolean:
1653    case Primitive::kPrimByte:
1654    case Primitive::kPrimChar:
1655    case Primitive::kPrimShort:
1656    case Primitive::kPrimInt:
1657    case Primitive::kPrimNot:
1658    case Primitive::kPrimFloat:
1659      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
1660      break;
1661
1662    case Primitive::kPrimLong:
1663    case Primitive::kPrimDouble:
1664      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
1665      break;
1666
1667    default:
1668      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
1669  }
1670}
1671
1672void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store ATTRIBUTE_UNUSED) {
1673}
1674
1675void LocationsBuilderX86::HandleCondition(HCondition* cond) {
1676  LocationSummary* locations =
1677      new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
1678  // Handle the long/FP comparisons made in instruction simplification.
1679  switch (cond->InputAt(0)->GetType()) {
1680    case Primitive::kPrimLong: {
1681      locations->SetInAt(0, Location::RequiresRegister());
1682      locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1)));
1683      if (cond->NeedsMaterialization()) {
1684        locations->SetOut(Location::RequiresRegister());
1685      }
1686      break;
1687    }
1688    case Primitive::kPrimFloat:
1689    case Primitive::kPrimDouble: {
1690      locations->SetInAt(0, Location::RequiresFpuRegister());
1691      locations->SetInAt(1, Location::RequiresFpuRegister());
1692      if (cond->NeedsMaterialization()) {
1693        locations->SetOut(Location::RequiresRegister());
1694      }
1695      break;
1696    }
1697    default:
1698      locations->SetInAt(0, Location::RequiresRegister());
1699      locations->SetInAt(1, Location::Any());
1700      if (cond->NeedsMaterialization()) {
1701        // We need a byte register.
1702        locations->SetOut(Location::RegisterLocation(ECX));
1703      }
1704      break;
1705  }
1706}
1707
1708void InstructionCodeGeneratorX86::HandleCondition(HCondition* cond) {
1709  if (!cond->NeedsMaterialization()) {
1710    return;
1711  }
1712
1713  LocationSummary* locations = cond->GetLocations();
1714  Location lhs = locations->InAt(0);
1715  Location rhs = locations->InAt(1);
1716  Register reg = locations->Out().AsRegister<Register>();
1717  NearLabel true_label, false_label;
1718
1719  switch (cond->InputAt(0)->GetType()) {
1720    default: {
1721      // Integer case.
1722
1723      // Clear output register: setb only sets the low byte.
1724      __ xorl(reg, reg);
1725
1726      if (rhs.IsRegister()) {
1727        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1728      } else if (rhs.IsConstant()) {
1729        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1730        if (constant == 0) {
1731          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1732        } else {
1733          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1734        }
1735      } else {
1736        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1737      }
1738      __ setb(X86Condition(cond->GetCondition()), reg);
1739      return;
1740    }
1741    case Primitive::kPrimLong:
1742      GenerateLongComparesAndJumps(cond, &true_label, &false_label);
1743      break;
1744    case Primitive::kPrimFloat:
1745      __ ucomiss(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1746      GenerateFPJumps(cond, &true_label, &false_label);
1747      break;
1748    case Primitive::kPrimDouble:
1749      __ ucomisd(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1750      GenerateFPJumps(cond, &true_label, &false_label);
1751      break;
1752  }
1753
1754  // Convert the jumps into the result.
1755  NearLabel done_label;
1756
1757  // False case: result = 0.
1758  __ Bind(&false_label);
1759  __ xorl(reg, reg);
1760  __ jmp(&done_label);
1761
1762  // True case: result = 1.
1763  __ Bind(&true_label);
1764  __ movl(reg, Immediate(1));
1765  __ Bind(&done_label);
1766}
1767
1768void LocationsBuilderX86::VisitEqual(HEqual* comp) {
1769  HandleCondition(comp);
1770}
1771
1772void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
1773  HandleCondition(comp);
1774}
1775
1776void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
1777  HandleCondition(comp);
1778}
1779
1780void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
1781  HandleCondition(comp);
1782}
1783
1784void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
1785  HandleCondition(comp);
1786}
1787
1788void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
1789  HandleCondition(comp);
1790}
1791
1792void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1793  HandleCondition(comp);
1794}
1795
1796void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1797  HandleCondition(comp);
1798}
1799
1800void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
1801  HandleCondition(comp);
1802}
1803
1804void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
1805  HandleCondition(comp);
1806}
1807
1808void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1809  HandleCondition(comp);
1810}
1811
1812void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1813  HandleCondition(comp);
1814}
1815
1816void LocationsBuilderX86::VisitBelow(HBelow* comp) {
1817  HandleCondition(comp);
1818}
1819
1820void InstructionCodeGeneratorX86::VisitBelow(HBelow* comp) {
1821  HandleCondition(comp);
1822}
1823
1824void LocationsBuilderX86::VisitBelowOrEqual(HBelowOrEqual* comp) {
1825  HandleCondition(comp);
1826}
1827
1828void InstructionCodeGeneratorX86::VisitBelowOrEqual(HBelowOrEqual* comp) {
1829  HandleCondition(comp);
1830}
1831
1832void LocationsBuilderX86::VisitAbove(HAbove* comp) {
1833  HandleCondition(comp);
1834}
1835
1836void InstructionCodeGeneratorX86::VisitAbove(HAbove* comp) {
1837  HandleCondition(comp);
1838}
1839
1840void LocationsBuilderX86::VisitAboveOrEqual(HAboveOrEqual* comp) {
1841  HandleCondition(comp);
1842}
1843
1844void InstructionCodeGeneratorX86::VisitAboveOrEqual(HAboveOrEqual* comp) {
1845  HandleCondition(comp);
1846}
1847
1848void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1849  LocationSummary* locations =
1850      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1851  locations->SetOut(Location::ConstantLocation(constant));
1852}
1853
1854void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) {
1855  // Will be generated at use site.
1856}
1857
1858void LocationsBuilderX86::VisitNullConstant(HNullConstant* constant) {
1859  LocationSummary* locations =
1860      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1861  locations->SetOut(Location::ConstantLocation(constant));
1862}
1863
1864void InstructionCodeGeneratorX86::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) {
1865  // Will be generated at use site.
1866}
1867
1868void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1869  LocationSummary* locations =
1870      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1871  locations->SetOut(Location::ConstantLocation(constant));
1872}
1873
1874void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) {
1875  // Will be generated at use site.
1876}
1877
1878void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1879  LocationSummary* locations =
1880      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1881  locations->SetOut(Location::ConstantLocation(constant));
1882}
1883
1884void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) {
1885  // Will be generated at use site.
1886}
1887
1888void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1889  LocationSummary* locations =
1890      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1891  locations->SetOut(Location::ConstantLocation(constant));
1892}
1893
1894void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) {
1895  // Will be generated at use site.
1896}
1897
1898void LocationsBuilderX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1899  memory_barrier->SetLocations(nullptr);
1900}
1901
1902void InstructionCodeGeneratorX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1903  codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1904}
1905
1906void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1907  ret->SetLocations(nullptr);
1908}
1909
1910void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) {
1911  codegen_->GenerateFrameExit();
1912}
1913
1914void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1915  LocationSummary* locations =
1916      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1917  switch (ret->InputAt(0)->GetType()) {
1918    case Primitive::kPrimBoolean:
1919    case Primitive::kPrimByte:
1920    case Primitive::kPrimChar:
1921    case Primitive::kPrimShort:
1922    case Primitive::kPrimInt:
1923    case Primitive::kPrimNot:
1924      locations->SetInAt(0, Location::RegisterLocation(EAX));
1925      break;
1926
1927    case Primitive::kPrimLong:
1928      locations->SetInAt(
1929          0, Location::RegisterPairLocation(EAX, EDX));
1930      break;
1931
1932    case Primitive::kPrimFloat:
1933    case Primitive::kPrimDouble:
1934      locations->SetInAt(
1935          0, Location::FpuRegisterLocation(XMM0));
1936      break;
1937
1938    default:
1939      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1940  }
1941}
1942
1943void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1944  if (kIsDebugBuild) {
1945    switch (ret->InputAt(0)->GetType()) {
1946      case Primitive::kPrimBoolean:
1947      case Primitive::kPrimByte:
1948      case Primitive::kPrimChar:
1949      case Primitive::kPrimShort:
1950      case Primitive::kPrimInt:
1951      case Primitive::kPrimNot:
1952        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1953        break;
1954
1955      case Primitive::kPrimLong:
1956        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1957        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1958        break;
1959
1960      case Primitive::kPrimFloat:
1961      case Primitive::kPrimDouble:
1962        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1963        break;
1964
1965      default:
1966        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1967    }
1968  }
1969  codegen_->GenerateFrameExit();
1970}
1971
1972void LocationsBuilderX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1973  // The trampoline uses the same calling convention as dex calling conventions,
1974  // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain
1975  // the method_idx.
1976  HandleInvoke(invoke);
1977}
1978
1979void InstructionCodeGeneratorX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1980  codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
1981}
1982
1983void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1984  // When we do not run baseline, explicit clinit checks triggered by static
1985  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1986  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1987
1988  IntrinsicLocationsBuilderX86 intrinsic(codegen_);
1989  if (intrinsic.TryDispatch(invoke)) {
1990    if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) {
1991      invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any());
1992    }
1993    return;
1994  }
1995
1996  HandleInvoke(invoke);
1997
1998  // For PC-relative dex cache the invoke has an extra input, the PC-relative address base.
1999  if (invoke->HasPcRelativeDexCache()) {
2000    invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister());
2001  }
2002
2003  if (codegen_->IsBaseline()) {
2004    // Baseline does not have enough registers if the current method also
2005    // needs a register. We therefore do not require a register for it, and let
2006    // the code generation of the invoke handle it.
2007    LocationSummary* locations = invoke->GetLocations();
2008    Location location = locations->InAt(invoke->GetSpecialInputIndex());
2009    if (location.IsUnallocated() && location.GetPolicy() == Location::kRequiresRegister) {
2010      locations->SetInAt(invoke->GetSpecialInputIndex(), Location::NoLocation());
2011    }
2012  }
2013}
2014
2015static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) {
2016  if (invoke->GetLocations()->Intrinsified()) {
2017    IntrinsicCodeGeneratorX86 intrinsic(codegen);
2018    intrinsic.Dispatch(invoke);
2019    return true;
2020  }
2021  return false;
2022}
2023
2024void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
2025  // When we do not run baseline, explicit clinit checks triggered by static
2026  // invokes must have been pruned by art::PrepareForRegisterAllocation.
2027  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
2028
2029  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
2030    return;
2031  }
2032
2033  LocationSummary* locations = invoke->GetLocations();
2034  codegen_->GenerateStaticOrDirectCall(
2035      invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
2036  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2037}
2038
2039void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
2040  IntrinsicLocationsBuilderX86 intrinsic(codegen_);
2041  if (intrinsic.TryDispatch(invoke)) {
2042    return;
2043  }
2044
2045  HandleInvoke(invoke);
2046}
2047
2048void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
2049  InvokeDexCallingConventionVisitorX86 calling_convention_visitor;
2050  CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
2051}
2052
2053void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
2054  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
2055    return;
2056  }
2057
2058  codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
2059  DCHECK(!codegen_->IsLeafMethod());
2060  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2061}
2062
2063void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
2064  // This call to HandleInvoke allocates a temporary (core) register
2065  // which is also used to transfer the hidden argument from FP to
2066  // core register.
2067  HandleInvoke(invoke);
2068  // Add the hidden argument.
2069  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
2070}
2071
2072void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
2073  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
2074  LocationSummary* locations = invoke->GetLocations();
2075  Register temp = locations->GetTemp(0).AsRegister<Register>();
2076  XmmRegister hidden_reg = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
2077  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
2078      invoke->GetImtIndex() % mirror::Class::kImtSize, kX86PointerSize).Uint32Value();
2079  Location receiver = locations->InAt(0);
2080  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
2081
2082  // Set the hidden argument. This is safe to do this here, as XMM7
2083  // won't be modified thereafter, before the `call` instruction.
2084  DCHECK_EQ(XMM7, hidden_reg);
2085  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
2086  __ movd(hidden_reg, temp);
2087
2088  if (receiver.IsStackSlot()) {
2089    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
2090    // /* HeapReference<Class> */ temp = temp->klass_
2091    __ movl(temp, Address(temp, class_offset));
2092  } else {
2093    // /* HeapReference<Class> */ temp = receiver->klass_
2094    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
2095  }
2096  codegen_->MaybeRecordImplicitNullCheck(invoke);
2097  // Instead of simply (possibly) unpoisoning `temp` here, we should
2098  // emit a read barrier for the previous class reference load.
2099  // However this is not required in practice, as this is an
2100  // intermediate/temporary reference and because the current
2101  // concurrent copying collector keeps the from-space memory
2102  // intact/accessible until the end of the marking phase (the
2103  // concurrent copying collector may not in the future).
2104  __ MaybeUnpoisonHeapReference(temp);
2105  // temp = temp->GetImtEntryAt(method_offset);
2106  __ movl(temp, Address(temp, method_offset));
2107  // call temp->GetEntryPoint();
2108  __ call(Address(temp,
2109                  ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
2110
2111  DCHECK(!codegen_->IsLeafMethod());
2112  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2113}
2114
2115void LocationsBuilderX86::VisitNeg(HNeg* neg) {
2116  LocationSummary* locations =
2117      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
2118  switch (neg->GetResultType()) {
2119    case Primitive::kPrimInt:
2120    case Primitive::kPrimLong:
2121      locations->SetInAt(0, Location::RequiresRegister());
2122      locations->SetOut(Location::SameAsFirstInput());
2123      break;
2124
2125    case Primitive::kPrimFloat:
2126      locations->SetInAt(0, Location::RequiresFpuRegister());
2127      locations->SetOut(Location::SameAsFirstInput());
2128      locations->AddTemp(Location::RequiresRegister());
2129      locations->AddTemp(Location::RequiresFpuRegister());
2130      break;
2131
2132    case Primitive::kPrimDouble:
2133      locations->SetInAt(0, Location::RequiresFpuRegister());
2134      locations->SetOut(Location::SameAsFirstInput());
2135      locations->AddTemp(Location::RequiresFpuRegister());
2136      break;
2137
2138    default:
2139      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
2140  }
2141}
2142
2143void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
2144  LocationSummary* locations = neg->GetLocations();
2145  Location out = locations->Out();
2146  Location in = locations->InAt(0);
2147  switch (neg->GetResultType()) {
2148    case Primitive::kPrimInt:
2149      DCHECK(in.IsRegister());
2150      DCHECK(in.Equals(out));
2151      __ negl(out.AsRegister<Register>());
2152      break;
2153
2154    case Primitive::kPrimLong:
2155      DCHECK(in.IsRegisterPair());
2156      DCHECK(in.Equals(out));
2157      __ negl(out.AsRegisterPairLow<Register>());
2158      // Negation is similar to subtraction from zero.  The least
2159      // significant byte triggers a borrow when it is different from
2160      // zero; to take it into account, add 1 to the most significant
2161      // byte if the carry flag (CF) is set to 1 after the first NEGL
2162      // operation.
2163      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
2164      __ negl(out.AsRegisterPairHigh<Register>());
2165      break;
2166
2167    case Primitive::kPrimFloat: {
2168      DCHECK(in.Equals(out));
2169      Register constant = locations->GetTemp(0).AsRegister<Register>();
2170      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
2171      // Implement float negation with an exclusive or with value
2172      // 0x80000000 (mask for bit 31, representing the sign of a
2173      // single-precision floating-point number).
2174      __ movl(constant, Immediate(INT32_C(0x80000000)));
2175      __ movd(mask, constant);
2176      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
2177      break;
2178    }
2179
2180    case Primitive::kPrimDouble: {
2181      DCHECK(in.Equals(out));
2182      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2183      // Implement double negation with an exclusive or with value
2184      // 0x8000000000000000 (mask for bit 63, representing the sign of
2185      // a double-precision floating-point number).
2186      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
2187      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
2188      break;
2189    }
2190
2191    default:
2192      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
2193  }
2194}
2195
2196void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
2197  Primitive::Type result_type = conversion->GetResultType();
2198  Primitive::Type input_type = conversion->GetInputType();
2199  DCHECK_NE(result_type, input_type);
2200
2201  // The float-to-long and double-to-long type conversions rely on a
2202  // call to the runtime.
2203  LocationSummary::CallKind call_kind =
2204      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
2205       && result_type == Primitive::kPrimLong)
2206      ? LocationSummary::kCall
2207      : LocationSummary::kNoCall;
2208  LocationSummary* locations =
2209      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
2210
2211  // The Java language does not allow treating boolean as an integral type but
2212  // our bit representation makes it safe.
2213
2214  switch (result_type) {
2215    case Primitive::kPrimByte:
2216      switch (input_type) {
2217        case Primitive::kPrimBoolean:
2218          // Boolean input is a result of code transformations.
2219        case Primitive::kPrimShort:
2220        case Primitive::kPrimInt:
2221        case Primitive::kPrimChar:
2222          // Processing a Dex `int-to-byte' instruction.
2223          locations->SetInAt(0, Location::ByteRegisterOrConstant(ECX, conversion->InputAt(0)));
2224          // Make the output overlap to please the register allocator. This greatly simplifies
2225          // the validation of the linear scan implementation
2226          locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
2227          break;
2228
2229        default:
2230          LOG(FATAL) << "Unexpected type conversion from " << input_type
2231                     << " to " << result_type;
2232      }
2233      break;
2234
2235    case Primitive::kPrimShort:
2236      switch (input_type) {
2237        case Primitive::kPrimBoolean:
2238          // Boolean input is a result of code transformations.
2239        case Primitive::kPrimByte:
2240        case Primitive::kPrimInt:
2241        case Primitive::kPrimChar:
2242          // Processing a Dex `int-to-short' instruction.
2243          locations->SetInAt(0, Location::Any());
2244          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2245          break;
2246
2247        default:
2248          LOG(FATAL) << "Unexpected type conversion from " << input_type
2249                     << " to " << result_type;
2250      }
2251      break;
2252
2253    case Primitive::kPrimInt:
2254      switch (input_type) {
2255        case Primitive::kPrimLong:
2256          // Processing a Dex `long-to-int' instruction.
2257          locations->SetInAt(0, Location::Any());
2258          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2259          break;
2260
2261        case Primitive::kPrimFloat:
2262          // Processing a Dex `float-to-int' instruction.
2263          locations->SetInAt(0, Location::RequiresFpuRegister());
2264          locations->SetOut(Location::RequiresRegister());
2265          locations->AddTemp(Location::RequiresFpuRegister());
2266          break;
2267
2268        case Primitive::kPrimDouble:
2269          // Processing a Dex `double-to-int' instruction.
2270          locations->SetInAt(0, Location::RequiresFpuRegister());
2271          locations->SetOut(Location::RequiresRegister());
2272          locations->AddTemp(Location::RequiresFpuRegister());
2273          break;
2274
2275        default:
2276          LOG(FATAL) << "Unexpected type conversion from " << input_type
2277                     << " to " << result_type;
2278      }
2279      break;
2280
2281    case Primitive::kPrimLong:
2282      switch (input_type) {
2283        case Primitive::kPrimBoolean:
2284          // Boolean input is a result of code transformations.
2285        case Primitive::kPrimByte:
2286        case Primitive::kPrimShort:
2287        case Primitive::kPrimInt:
2288        case Primitive::kPrimChar:
2289          // Processing a Dex `int-to-long' instruction.
2290          locations->SetInAt(0, Location::RegisterLocation(EAX));
2291          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2292          break;
2293
2294        case Primitive::kPrimFloat:
2295        case Primitive::kPrimDouble: {
2296          // Processing a Dex `float-to-long' or 'double-to-long' instruction.
2297          InvokeRuntimeCallingConvention calling_convention;
2298          XmmRegister parameter = calling_convention.GetFpuRegisterAt(0);
2299          locations->SetInAt(0, Location::FpuRegisterLocation(parameter));
2300
2301          // The runtime helper puts the result in EAX, EDX.
2302          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2303        }
2304        break;
2305
2306        default:
2307          LOG(FATAL) << "Unexpected type conversion from " << input_type
2308                     << " to " << result_type;
2309      }
2310      break;
2311
2312    case Primitive::kPrimChar:
2313      switch (input_type) {
2314        case Primitive::kPrimBoolean:
2315          // Boolean input is a result of code transformations.
2316        case Primitive::kPrimByte:
2317        case Primitive::kPrimShort:
2318        case Primitive::kPrimInt:
2319          // Processing a Dex `int-to-char' instruction.
2320          locations->SetInAt(0, Location::Any());
2321          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2322          break;
2323
2324        default:
2325          LOG(FATAL) << "Unexpected type conversion from " << input_type
2326                     << " to " << result_type;
2327      }
2328      break;
2329
2330    case Primitive::kPrimFloat:
2331      switch (input_type) {
2332        case Primitive::kPrimBoolean:
2333          // Boolean input is a result of code transformations.
2334        case Primitive::kPrimByte:
2335        case Primitive::kPrimShort:
2336        case Primitive::kPrimInt:
2337        case Primitive::kPrimChar:
2338          // Processing a Dex `int-to-float' instruction.
2339          locations->SetInAt(0, Location::RequiresRegister());
2340          locations->SetOut(Location::RequiresFpuRegister());
2341          break;
2342
2343        case Primitive::kPrimLong:
2344          // Processing a Dex `long-to-float' instruction.
2345          locations->SetInAt(0, Location::Any());
2346          locations->SetOut(Location::Any());
2347          break;
2348
2349        case Primitive::kPrimDouble:
2350          // Processing a Dex `double-to-float' instruction.
2351          locations->SetInAt(0, Location::RequiresFpuRegister());
2352          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2353          break;
2354
2355        default:
2356          LOG(FATAL) << "Unexpected type conversion from " << input_type
2357                     << " to " << result_type;
2358      };
2359      break;
2360
2361    case Primitive::kPrimDouble:
2362      switch (input_type) {
2363        case Primitive::kPrimBoolean:
2364          // Boolean input is a result of code transformations.
2365        case Primitive::kPrimByte:
2366        case Primitive::kPrimShort:
2367        case Primitive::kPrimInt:
2368        case Primitive::kPrimChar:
2369          // Processing a Dex `int-to-double' instruction.
2370          locations->SetInAt(0, Location::RequiresRegister());
2371          locations->SetOut(Location::RequiresFpuRegister());
2372          break;
2373
2374        case Primitive::kPrimLong:
2375          // Processing a Dex `long-to-double' instruction.
2376          locations->SetInAt(0, Location::Any());
2377          locations->SetOut(Location::Any());
2378          break;
2379
2380        case Primitive::kPrimFloat:
2381          // Processing a Dex `float-to-double' instruction.
2382          locations->SetInAt(0, Location::RequiresFpuRegister());
2383          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2384          break;
2385
2386        default:
2387          LOG(FATAL) << "Unexpected type conversion from " << input_type
2388                     << " to " << result_type;
2389      }
2390      break;
2391
2392    default:
2393      LOG(FATAL) << "Unexpected type conversion from " << input_type
2394                 << " to " << result_type;
2395  }
2396}
2397
2398void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
2399  LocationSummary* locations = conversion->GetLocations();
2400  Location out = locations->Out();
2401  Location in = locations->InAt(0);
2402  Primitive::Type result_type = conversion->GetResultType();
2403  Primitive::Type input_type = conversion->GetInputType();
2404  DCHECK_NE(result_type, input_type);
2405  switch (result_type) {
2406    case Primitive::kPrimByte:
2407      switch (input_type) {
2408        case Primitive::kPrimBoolean:
2409          // Boolean input is a result of code transformations.
2410        case Primitive::kPrimShort:
2411        case Primitive::kPrimInt:
2412        case Primitive::kPrimChar:
2413          // Processing a Dex `int-to-byte' instruction.
2414          if (in.IsRegister()) {
2415            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
2416          } else {
2417            DCHECK(in.GetConstant()->IsIntConstant());
2418            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2419            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
2420          }
2421          break;
2422
2423        default:
2424          LOG(FATAL) << "Unexpected type conversion from " << input_type
2425                     << " to " << result_type;
2426      }
2427      break;
2428
2429    case Primitive::kPrimShort:
2430      switch (input_type) {
2431        case Primitive::kPrimBoolean:
2432          // Boolean input is a result of code transformations.
2433        case Primitive::kPrimByte:
2434        case Primitive::kPrimInt:
2435        case Primitive::kPrimChar:
2436          // Processing a Dex `int-to-short' instruction.
2437          if (in.IsRegister()) {
2438            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2439          } else if (in.IsStackSlot()) {
2440            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2441          } else {
2442            DCHECK(in.GetConstant()->IsIntConstant());
2443            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2444            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
2445          }
2446          break;
2447
2448        default:
2449          LOG(FATAL) << "Unexpected type conversion from " << input_type
2450                     << " to " << result_type;
2451      }
2452      break;
2453
2454    case Primitive::kPrimInt:
2455      switch (input_type) {
2456        case Primitive::kPrimLong:
2457          // Processing a Dex `long-to-int' instruction.
2458          if (in.IsRegisterPair()) {
2459            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
2460          } else if (in.IsDoubleStackSlot()) {
2461            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2462          } else {
2463            DCHECK(in.IsConstant());
2464            DCHECK(in.GetConstant()->IsLongConstant());
2465            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
2466            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
2467          }
2468          break;
2469
2470        case Primitive::kPrimFloat: {
2471          // Processing a Dex `float-to-int' instruction.
2472          XmmRegister input = in.AsFpuRegister<XmmRegister>();
2473          Register output = out.AsRegister<Register>();
2474          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2475          NearLabel done, nan;
2476
2477          __ movl(output, Immediate(kPrimIntMax));
2478          // temp = int-to-float(output)
2479          __ cvtsi2ss(temp, output);
2480          // if input >= temp goto done
2481          __ comiss(input, temp);
2482          __ j(kAboveEqual, &done);
2483          // if input == NaN goto nan
2484          __ j(kUnordered, &nan);
2485          // output = float-to-int-truncate(input)
2486          __ cvttss2si(output, input);
2487          __ jmp(&done);
2488          __ Bind(&nan);
2489          //  output = 0
2490          __ xorl(output, output);
2491          __ Bind(&done);
2492          break;
2493        }
2494
2495        case Primitive::kPrimDouble: {
2496          // Processing a Dex `double-to-int' instruction.
2497          XmmRegister input = in.AsFpuRegister<XmmRegister>();
2498          Register output = out.AsRegister<Register>();
2499          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2500          NearLabel done, nan;
2501
2502          __ movl(output, Immediate(kPrimIntMax));
2503          // temp = int-to-double(output)
2504          __ cvtsi2sd(temp, output);
2505          // if input >= temp goto done
2506          __ comisd(input, temp);
2507          __ j(kAboveEqual, &done);
2508          // if input == NaN goto nan
2509          __ j(kUnordered, &nan);
2510          // output = double-to-int-truncate(input)
2511          __ cvttsd2si(output, input);
2512          __ jmp(&done);
2513          __ Bind(&nan);
2514          //  output = 0
2515          __ xorl(output, output);
2516          __ Bind(&done);
2517          break;
2518        }
2519
2520        default:
2521          LOG(FATAL) << "Unexpected type conversion from " << input_type
2522                     << " to " << result_type;
2523      }
2524      break;
2525
2526    case Primitive::kPrimLong:
2527      switch (input_type) {
2528        case Primitive::kPrimBoolean:
2529          // Boolean input is a result of code transformations.
2530        case Primitive::kPrimByte:
2531        case Primitive::kPrimShort:
2532        case Primitive::kPrimInt:
2533        case Primitive::kPrimChar:
2534          // Processing a Dex `int-to-long' instruction.
2535          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
2536          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
2537          DCHECK_EQ(in.AsRegister<Register>(), EAX);
2538          __ cdq();
2539          break;
2540
2541        case Primitive::kPrimFloat:
2542          // Processing a Dex `float-to-long' instruction.
2543          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pF2l),
2544                                  conversion,
2545                                  conversion->GetDexPc(),
2546                                  nullptr);
2547          CheckEntrypointTypes<kQuickF2l, int64_t, float>();
2548          break;
2549
2550        case Primitive::kPrimDouble:
2551          // Processing a Dex `double-to-long' instruction.
2552          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pD2l),
2553                                  conversion,
2554                                  conversion->GetDexPc(),
2555                                  nullptr);
2556          CheckEntrypointTypes<kQuickD2l, int64_t, double>();
2557          break;
2558
2559        default:
2560          LOG(FATAL) << "Unexpected type conversion from " << input_type
2561                     << " to " << result_type;
2562      }
2563      break;
2564
2565    case Primitive::kPrimChar:
2566      switch (input_type) {
2567        case Primitive::kPrimBoolean:
2568          // Boolean input is a result of code transformations.
2569        case Primitive::kPrimByte:
2570        case Primitive::kPrimShort:
2571        case Primitive::kPrimInt:
2572          // Processing a Dex `Process a Dex `int-to-char'' instruction.
2573          if (in.IsRegister()) {
2574            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2575          } else if (in.IsStackSlot()) {
2576            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2577          } else {
2578            DCHECK(in.GetConstant()->IsIntConstant());
2579            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2580            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
2581          }
2582          break;
2583
2584        default:
2585          LOG(FATAL) << "Unexpected type conversion from " << input_type
2586                     << " to " << result_type;
2587      }
2588      break;
2589
2590    case Primitive::kPrimFloat:
2591      switch (input_type) {
2592        case Primitive::kPrimBoolean:
2593          // Boolean input is a result of code transformations.
2594        case Primitive::kPrimByte:
2595        case Primitive::kPrimShort:
2596        case Primitive::kPrimInt:
2597        case Primitive::kPrimChar:
2598          // Processing a Dex `int-to-float' instruction.
2599          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2600          break;
2601
2602        case Primitive::kPrimLong: {
2603          // Processing a Dex `long-to-float' instruction.
2604          size_t adjustment = 0;
2605
2606          // Create stack space for the call to
2607          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstps below.
2608          // TODO: enhance register allocator to ask for stack temporaries.
2609          if (!in.IsDoubleStackSlot() || !out.IsStackSlot()) {
2610            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2611            __ subl(ESP, Immediate(adjustment));
2612          }
2613
2614          // Load the value to the FP stack, using temporaries if needed.
2615          PushOntoFPStack(in, 0, adjustment, false, true);
2616
2617          if (out.IsStackSlot()) {
2618            __ fstps(Address(ESP, out.GetStackIndex() + adjustment));
2619          } else {
2620            __ fstps(Address(ESP, 0));
2621            Location stack_temp = Location::StackSlot(0);
2622            codegen_->Move32(out, stack_temp);
2623          }
2624
2625          // Remove the temporary stack space we allocated.
2626          if (adjustment != 0) {
2627            __ addl(ESP, Immediate(adjustment));
2628          }
2629          break;
2630        }
2631
2632        case Primitive::kPrimDouble:
2633          // Processing a Dex `double-to-float' instruction.
2634          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2635          break;
2636
2637        default:
2638          LOG(FATAL) << "Unexpected type conversion from " << input_type
2639                     << " to " << result_type;
2640      };
2641      break;
2642
2643    case Primitive::kPrimDouble:
2644      switch (input_type) {
2645        case Primitive::kPrimBoolean:
2646          // Boolean input is a result of code transformations.
2647        case Primitive::kPrimByte:
2648        case Primitive::kPrimShort:
2649        case Primitive::kPrimInt:
2650        case Primitive::kPrimChar:
2651          // Processing a Dex `int-to-double' instruction.
2652          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2653          break;
2654
2655        case Primitive::kPrimLong: {
2656          // Processing a Dex `long-to-double' instruction.
2657          size_t adjustment = 0;
2658
2659          // Create stack space for the call to
2660          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstpl below.
2661          // TODO: enhance register allocator to ask for stack temporaries.
2662          if (!in.IsDoubleStackSlot() || !out.IsDoubleStackSlot()) {
2663            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2664            __ subl(ESP, Immediate(adjustment));
2665          }
2666
2667          // Load the value to the FP stack, using temporaries if needed.
2668          PushOntoFPStack(in, 0, adjustment, false, true);
2669
2670          if (out.IsDoubleStackSlot()) {
2671            __ fstpl(Address(ESP, out.GetStackIndex() + adjustment));
2672          } else {
2673            __ fstpl(Address(ESP, 0));
2674            Location stack_temp = Location::DoubleStackSlot(0);
2675            codegen_->Move64(out, stack_temp);
2676          }
2677
2678          // Remove the temporary stack space we allocated.
2679          if (adjustment != 0) {
2680            __ addl(ESP, Immediate(adjustment));
2681          }
2682          break;
2683        }
2684
2685        case Primitive::kPrimFloat:
2686          // Processing a Dex `float-to-double' instruction.
2687          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2688          break;
2689
2690        default:
2691          LOG(FATAL) << "Unexpected type conversion from " << input_type
2692                     << " to " << result_type;
2693      };
2694      break;
2695
2696    default:
2697      LOG(FATAL) << "Unexpected type conversion from " << input_type
2698                 << " to " << result_type;
2699  }
2700}
2701
2702void LocationsBuilderX86::VisitAdd(HAdd* add) {
2703  LocationSummary* locations =
2704      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
2705  switch (add->GetResultType()) {
2706    case Primitive::kPrimInt: {
2707      locations->SetInAt(0, Location::RequiresRegister());
2708      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
2709      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2710      break;
2711    }
2712
2713    case Primitive::kPrimLong: {
2714      locations->SetInAt(0, Location::RequiresRegister());
2715      locations->SetInAt(1, Location::Any());
2716      locations->SetOut(Location::SameAsFirstInput());
2717      break;
2718    }
2719
2720    case Primitive::kPrimFloat:
2721    case Primitive::kPrimDouble: {
2722      locations->SetInAt(0, Location::RequiresFpuRegister());
2723      locations->SetInAt(1, Location::Any());
2724      locations->SetOut(Location::SameAsFirstInput());
2725      break;
2726    }
2727
2728    default:
2729      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2730      break;
2731  }
2732}
2733
2734void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
2735  LocationSummary* locations = add->GetLocations();
2736  Location first = locations->InAt(0);
2737  Location second = locations->InAt(1);
2738  Location out = locations->Out();
2739
2740  switch (add->GetResultType()) {
2741    case Primitive::kPrimInt: {
2742      if (second.IsRegister()) {
2743        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2744          __ addl(out.AsRegister<Register>(), second.AsRegister<Register>());
2745        } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
2746          __ addl(out.AsRegister<Register>(), first.AsRegister<Register>());
2747        } else {
2748          __ leal(out.AsRegister<Register>(), Address(
2749              first.AsRegister<Register>(), second.AsRegister<Register>(), TIMES_1, 0));
2750          }
2751      } else if (second.IsConstant()) {
2752        int32_t value = second.GetConstant()->AsIntConstant()->GetValue();
2753        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2754          __ addl(out.AsRegister<Register>(), Immediate(value));
2755        } else {
2756          __ leal(out.AsRegister<Register>(), Address(first.AsRegister<Register>(), value));
2757        }
2758      } else {
2759        DCHECK(first.Equals(locations->Out()));
2760        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2761      }
2762      break;
2763    }
2764
2765    case Primitive::kPrimLong: {
2766      if (second.IsRegisterPair()) {
2767        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2768        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2769      } else if (second.IsDoubleStackSlot()) {
2770        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2771        __ adcl(first.AsRegisterPairHigh<Register>(),
2772                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2773      } else {
2774        DCHECK(second.IsConstant()) << second;
2775        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2776        __ addl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2777        __ adcl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2778      }
2779      break;
2780    }
2781
2782    case Primitive::kPrimFloat: {
2783      if (second.IsFpuRegister()) {
2784        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2785      } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2786        HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2787        DCHECK(!const_area->NeedsMaterialization());
2788        __ addss(first.AsFpuRegister<XmmRegister>(),
2789                 codegen_->LiteralFloatAddress(
2790                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2791                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2792      } else {
2793        DCHECK(second.IsStackSlot());
2794        __ addss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2795      }
2796      break;
2797    }
2798
2799    case Primitive::kPrimDouble: {
2800      if (second.IsFpuRegister()) {
2801        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2802      } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2803        HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2804        DCHECK(!const_area->NeedsMaterialization());
2805        __ addsd(first.AsFpuRegister<XmmRegister>(),
2806                 codegen_->LiteralDoubleAddress(
2807                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2808                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2809      } else {
2810        DCHECK(second.IsDoubleStackSlot());
2811        __ addsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2812      }
2813      break;
2814    }
2815
2816    default:
2817      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2818  }
2819}
2820
2821void LocationsBuilderX86::VisitSub(HSub* sub) {
2822  LocationSummary* locations =
2823      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2824  switch (sub->GetResultType()) {
2825    case Primitive::kPrimInt:
2826    case Primitive::kPrimLong: {
2827      locations->SetInAt(0, Location::RequiresRegister());
2828      locations->SetInAt(1, Location::Any());
2829      locations->SetOut(Location::SameAsFirstInput());
2830      break;
2831    }
2832    case Primitive::kPrimFloat:
2833    case Primitive::kPrimDouble: {
2834      locations->SetInAt(0, Location::RequiresFpuRegister());
2835      locations->SetInAt(1, Location::Any());
2836      locations->SetOut(Location::SameAsFirstInput());
2837      break;
2838    }
2839
2840    default:
2841      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2842  }
2843}
2844
2845void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
2846  LocationSummary* locations = sub->GetLocations();
2847  Location first = locations->InAt(0);
2848  Location second = locations->InAt(1);
2849  DCHECK(first.Equals(locations->Out()));
2850  switch (sub->GetResultType()) {
2851    case Primitive::kPrimInt: {
2852      if (second.IsRegister()) {
2853        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
2854      } else if (second.IsConstant()) {
2855        __ subl(first.AsRegister<Register>(),
2856                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2857      } else {
2858        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2859      }
2860      break;
2861    }
2862
2863    case Primitive::kPrimLong: {
2864      if (second.IsRegisterPair()) {
2865        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2866        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2867      } else if (second.IsDoubleStackSlot()) {
2868        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2869        __ sbbl(first.AsRegisterPairHigh<Register>(),
2870                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2871      } else {
2872        DCHECK(second.IsConstant()) << second;
2873        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2874        __ subl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2875        __ sbbl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2876      }
2877      break;
2878    }
2879
2880    case Primitive::kPrimFloat: {
2881      if (second.IsFpuRegister()) {
2882        __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2883      } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2884        HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2885        DCHECK(!const_area->NeedsMaterialization());
2886        __ subss(first.AsFpuRegister<XmmRegister>(),
2887                 codegen_->LiteralFloatAddress(
2888                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2889                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2890      } else {
2891        DCHECK(second.IsStackSlot());
2892        __ subss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2893      }
2894      break;
2895    }
2896
2897    case Primitive::kPrimDouble: {
2898      if (second.IsFpuRegister()) {
2899        __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2900      } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2901        HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2902        DCHECK(!const_area->NeedsMaterialization());
2903        __ subsd(first.AsFpuRegister<XmmRegister>(),
2904                 codegen_->LiteralDoubleAddress(
2905                     const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2906                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2907      } else {
2908        DCHECK(second.IsDoubleStackSlot());
2909        __ subsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2910      }
2911      break;
2912    }
2913
2914    default:
2915      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2916  }
2917}
2918
2919void LocationsBuilderX86::VisitMul(HMul* mul) {
2920  LocationSummary* locations =
2921      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2922  switch (mul->GetResultType()) {
2923    case Primitive::kPrimInt:
2924      locations->SetInAt(0, Location::RequiresRegister());
2925      locations->SetInAt(1, Location::Any());
2926      if (mul->InputAt(1)->IsIntConstant()) {
2927        // Can use 3 operand multiply.
2928        locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2929      } else {
2930        locations->SetOut(Location::SameAsFirstInput());
2931      }
2932      break;
2933    case Primitive::kPrimLong: {
2934      locations->SetInAt(0, Location::RequiresRegister());
2935      locations->SetInAt(1, Location::Any());
2936      locations->SetOut(Location::SameAsFirstInput());
2937      // Needed for imul on 32bits with 64bits output.
2938      locations->AddTemp(Location::RegisterLocation(EAX));
2939      locations->AddTemp(Location::RegisterLocation(EDX));
2940      break;
2941    }
2942    case Primitive::kPrimFloat:
2943    case Primitive::kPrimDouble: {
2944      locations->SetInAt(0, Location::RequiresFpuRegister());
2945      locations->SetInAt(1, Location::Any());
2946      locations->SetOut(Location::SameAsFirstInput());
2947      break;
2948    }
2949
2950    default:
2951      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2952  }
2953}
2954
2955void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2956  LocationSummary* locations = mul->GetLocations();
2957  Location first = locations->InAt(0);
2958  Location second = locations->InAt(1);
2959  Location out = locations->Out();
2960
2961  switch (mul->GetResultType()) {
2962    case Primitive::kPrimInt:
2963      // The constant may have ended up in a register, so test explicitly to avoid
2964      // problems where the output may not be the same as the first operand.
2965      if (mul->InputAt(1)->IsIntConstant()) {
2966        Immediate imm(mul->InputAt(1)->AsIntConstant()->GetValue());
2967        __ imull(out.AsRegister<Register>(), first.AsRegister<Register>(), imm);
2968      } else if (second.IsRegister()) {
2969        DCHECK(first.Equals(out));
2970        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
2971      } else {
2972        DCHECK(second.IsStackSlot());
2973        DCHECK(first.Equals(out));
2974        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2975      }
2976      break;
2977
2978    case Primitive::kPrimLong: {
2979      Register in1_hi = first.AsRegisterPairHigh<Register>();
2980      Register in1_lo = first.AsRegisterPairLow<Register>();
2981      Register eax = locations->GetTemp(0).AsRegister<Register>();
2982      Register edx = locations->GetTemp(1).AsRegister<Register>();
2983
2984      DCHECK_EQ(EAX, eax);
2985      DCHECK_EQ(EDX, edx);
2986
2987      // input: in1 - 64 bits, in2 - 64 bits.
2988      // output: in1
2989      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2990      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2991      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
2992      if (second.IsConstant()) {
2993        DCHECK(second.GetConstant()->IsLongConstant());
2994
2995        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2996        int32_t low_value = Low32Bits(value);
2997        int32_t high_value = High32Bits(value);
2998        Immediate low(low_value);
2999        Immediate high(high_value);
3000
3001        __ movl(eax, high);
3002        // eax <- in1.lo * in2.hi
3003        __ imull(eax, in1_lo);
3004        // in1.hi <- in1.hi * in2.lo
3005        __ imull(in1_hi, low);
3006        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
3007        __ addl(in1_hi, eax);
3008        // move in2_lo to eax to prepare for double precision
3009        __ movl(eax, low);
3010        // edx:eax <- in1.lo * in2.lo
3011        __ mull(in1_lo);
3012        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
3013        __ addl(in1_hi, edx);
3014        // in1.lo <- (in1.lo * in2.lo)[31:0];
3015        __ movl(in1_lo, eax);
3016      } else if (second.IsRegisterPair()) {
3017        Register in2_hi = second.AsRegisterPairHigh<Register>();
3018        Register in2_lo = second.AsRegisterPairLow<Register>();
3019
3020        __ movl(eax, in2_hi);
3021        // eax <- in1.lo * in2.hi
3022        __ imull(eax, in1_lo);
3023        // in1.hi <- in1.hi * in2.lo
3024        __ imull(in1_hi, in2_lo);
3025        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
3026        __ addl(in1_hi, eax);
3027        // move in1_lo to eax to prepare for double precision
3028        __ movl(eax, in1_lo);
3029        // edx:eax <- in1.lo * in2.lo
3030        __ mull(in2_lo);
3031        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
3032        __ addl(in1_hi, edx);
3033        // in1.lo <- (in1.lo * in2.lo)[31:0];
3034        __ movl(in1_lo, eax);
3035      } else {
3036        DCHECK(second.IsDoubleStackSlot()) << second;
3037        Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
3038        Address in2_lo(ESP, second.GetStackIndex());
3039
3040        __ movl(eax, in2_hi);
3041        // eax <- in1.lo * in2.hi
3042        __ imull(eax, in1_lo);
3043        // in1.hi <- in1.hi * in2.lo
3044        __ imull(in1_hi, in2_lo);
3045        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
3046        __ addl(in1_hi, eax);
3047        // move in1_lo to eax to prepare for double precision
3048        __ movl(eax, in1_lo);
3049        // edx:eax <- in1.lo * in2.lo
3050        __ mull(in2_lo);
3051        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
3052        __ addl(in1_hi, edx);
3053        // in1.lo <- (in1.lo * in2.lo)[31:0];
3054        __ movl(in1_lo, eax);
3055      }
3056
3057      break;
3058    }
3059
3060    case Primitive::kPrimFloat: {
3061      DCHECK(first.Equals(locations->Out()));
3062      if (second.IsFpuRegister()) {
3063        __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3064      } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
3065        HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
3066        DCHECK(!const_area->NeedsMaterialization());
3067        __ mulss(first.AsFpuRegister<XmmRegister>(),
3068                 codegen_->LiteralFloatAddress(
3069                     const_area->GetConstant()->AsFloatConstant()->GetValue(),
3070                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3071      } else {
3072        DCHECK(second.IsStackSlot());
3073        __ mulss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3074      }
3075      break;
3076    }
3077
3078    case Primitive::kPrimDouble: {
3079      DCHECK(first.Equals(locations->Out()));
3080      if (second.IsFpuRegister()) {
3081        __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3082      } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
3083        HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
3084        DCHECK(!const_area->NeedsMaterialization());
3085        __ mulsd(first.AsFpuRegister<XmmRegister>(),
3086                 codegen_->LiteralDoubleAddress(
3087                     const_area->GetConstant()->AsDoubleConstant()->GetValue(),
3088                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3089      } else {
3090        DCHECK(second.IsDoubleStackSlot());
3091        __ mulsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3092      }
3093      break;
3094    }
3095
3096    default:
3097      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
3098  }
3099}
3100
3101void InstructionCodeGeneratorX86::PushOntoFPStack(Location source,
3102                                                  uint32_t temp_offset,
3103                                                  uint32_t stack_adjustment,
3104                                                  bool is_fp,
3105                                                  bool is_wide) {
3106  if (source.IsStackSlot()) {
3107    DCHECK(!is_wide);
3108    if (is_fp) {
3109      __ flds(Address(ESP, source.GetStackIndex() + stack_adjustment));
3110    } else {
3111      __ filds(Address(ESP, source.GetStackIndex() + stack_adjustment));
3112    }
3113  } else if (source.IsDoubleStackSlot()) {
3114    DCHECK(is_wide);
3115    if (is_fp) {
3116      __ fldl(Address(ESP, source.GetStackIndex() + stack_adjustment));
3117    } else {
3118      __ fildl(Address(ESP, source.GetStackIndex() + stack_adjustment));
3119    }
3120  } else {
3121    // Write the value to the temporary location on the stack and load to FP stack.
3122    if (!is_wide) {
3123      Location stack_temp = Location::StackSlot(temp_offset);
3124      codegen_->Move32(stack_temp, source);
3125      if (is_fp) {
3126        __ flds(Address(ESP, temp_offset));
3127      } else {
3128        __ filds(Address(ESP, temp_offset));
3129      }
3130    } else {
3131      Location stack_temp = Location::DoubleStackSlot(temp_offset);
3132      codegen_->Move64(stack_temp, source);
3133      if (is_fp) {
3134        __ fldl(Address(ESP, temp_offset));
3135      } else {
3136        __ fildl(Address(ESP, temp_offset));
3137      }
3138    }
3139  }
3140}
3141
3142void InstructionCodeGeneratorX86::GenerateRemFP(HRem *rem) {
3143  Primitive::Type type = rem->GetResultType();
3144  bool is_float = type == Primitive::kPrimFloat;
3145  size_t elem_size = Primitive::ComponentSize(type);
3146  LocationSummary* locations = rem->GetLocations();
3147  Location first = locations->InAt(0);
3148  Location second = locations->InAt(1);
3149  Location out = locations->Out();
3150
3151  // Create stack space for 2 elements.
3152  // TODO: enhance register allocator to ask for stack temporaries.
3153  __ subl(ESP, Immediate(2 * elem_size));
3154
3155  // Load the values to the FP stack in reverse order, using temporaries if needed.
3156  const bool is_wide = !is_float;
3157  PushOntoFPStack(second, elem_size, 2 * elem_size, /* is_fp */ true, is_wide);
3158  PushOntoFPStack(first, 0, 2 * elem_size, /* is_fp */ true, is_wide);
3159
3160  // Loop doing FPREM until we stabilize.
3161  NearLabel retry;
3162  __ Bind(&retry);
3163  __ fprem();
3164
3165  // Move FP status to AX.
3166  __ fstsw();
3167
3168  // And see if the argument reduction is complete. This is signaled by the
3169  // C2 FPU flag bit set to 0.
3170  __ andl(EAX, Immediate(kC2ConditionMask));
3171  __ j(kNotEqual, &retry);
3172
3173  // We have settled on the final value. Retrieve it into an XMM register.
3174  // Store FP top of stack to real stack.
3175  if (is_float) {
3176    __ fsts(Address(ESP, 0));
3177  } else {
3178    __ fstl(Address(ESP, 0));
3179  }
3180
3181  // Pop the 2 items from the FP stack.
3182  __ fucompp();
3183
3184  // Load the value from the stack into an XMM register.
3185  DCHECK(out.IsFpuRegister()) << out;
3186  if (is_float) {
3187    __ movss(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
3188  } else {
3189    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
3190  }
3191
3192  // And remove the temporary stack space we allocated.
3193  __ addl(ESP, Immediate(2 * elem_size));
3194}
3195
3196
3197void InstructionCodeGeneratorX86::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
3198  DCHECK(instruction->IsDiv() || instruction->IsRem());
3199
3200  LocationSummary* locations = instruction->GetLocations();
3201  DCHECK(locations->InAt(1).IsConstant());
3202  DCHECK(locations->InAt(1).GetConstant()->IsIntConstant());
3203
3204  Register out_register = locations->Out().AsRegister<Register>();
3205  Register input_register = locations->InAt(0).AsRegister<Register>();
3206  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3207
3208  DCHECK(imm == 1 || imm == -1);
3209
3210  if (instruction->IsRem()) {
3211    __ xorl(out_register, out_register);
3212  } else {
3213    __ movl(out_register, input_register);
3214    if (imm == -1) {
3215      __ negl(out_register);
3216    }
3217  }
3218}
3219
3220
3221void InstructionCodeGeneratorX86::DivByPowerOfTwo(HDiv* instruction) {
3222  LocationSummary* locations = instruction->GetLocations();
3223
3224  Register out_register = locations->Out().AsRegister<Register>();
3225  Register input_register = locations->InAt(0).AsRegister<Register>();
3226  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3227  DCHECK(IsPowerOfTwo(AbsOrMin(imm)));
3228  uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm));
3229
3230  Register num = locations->GetTemp(0).AsRegister<Register>();
3231
3232  __ leal(num, Address(input_register, abs_imm - 1));
3233  __ testl(input_register, input_register);
3234  __ cmovl(kGreaterEqual, num, input_register);
3235  int shift = CTZ(imm);
3236  __ sarl(num, Immediate(shift));
3237
3238  if (imm < 0) {
3239    __ negl(num);
3240  }
3241
3242  __ movl(out_register, num);
3243}
3244
3245void InstructionCodeGeneratorX86::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
3246  DCHECK(instruction->IsDiv() || instruction->IsRem());
3247
3248  LocationSummary* locations = instruction->GetLocations();
3249  int imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3250
3251  Register eax = locations->InAt(0).AsRegister<Register>();
3252  Register out = locations->Out().AsRegister<Register>();
3253  Register num;
3254  Register edx;
3255
3256  if (instruction->IsDiv()) {
3257    edx = locations->GetTemp(0).AsRegister<Register>();
3258    num = locations->GetTemp(1).AsRegister<Register>();
3259  } else {
3260    edx = locations->Out().AsRegister<Register>();
3261    num = locations->GetTemp(0).AsRegister<Register>();
3262  }
3263
3264  DCHECK_EQ(EAX, eax);
3265  DCHECK_EQ(EDX, edx);
3266  if (instruction->IsDiv()) {
3267    DCHECK_EQ(EAX, out);
3268  } else {
3269    DCHECK_EQ(EDX, out);
3270  }
3271
3272  int64_t magic;
3273  int shift;
3274  CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
3275
3276  NearLabel ndiv;
3277  NearLabel end;
3278  // If numerator is 0, the result is 0, no computation needed.
3279  __ testl(eax, eax);
3280  __ j(kNotEqual, &ndiv);
3281
3282  __ xorl(out, out);
3283  __ jmp(&end);
3284
3285  __ Bind(&ndiv);
3286
3287  // Save the numerator.
3288  __ movl(num, eax);
3289
3290  // EAX = magic
3291  __ movl(eax, Immediate(magic));
3292
3293  // EDX:EAX = magic * numerator
3294  __ imull(num);
3295
3296  if (imm > 0 && magic < 0) {
3297    // EDX += num
3298    __ addl(edx, num);
3299  } else if (imm < 0 && magic > 0) {
3300    __ subl(edx, num);
3301  }
3302
3303  // Shift if needed.
3304  if (shift != 0) {
3305    __ sarl(edx, Immediate(shift));
3306  }
3307
3308  // EDX += 1 if EDX < 0
3309  __ movl(eax, edx);
3310  __ shrl(edx, Immediate(31));
3311  __ addl(edx, eax);
3312
3313  if (instruction->IsRem()) {
3314    __ movl(eax, num);
3315    __ imull(edx, Immediate(imm));
3316    __ subl(eax, edx);
3317    __ movl(edx, eax);
3318  } else {
3319    __ movl(eax, edx);
3320  }
3321  __ Bind(&end);
3322}
3323
3324void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
3325  DCHECK(instruction->IsDiv() || instruction->IsRem());
3326
3327  LocationSummary* locations = instruction->GetLocations();
3328  Location out = locations->Out();
3329  Location first = locations->InAt(0);
3330  Location second = locations->InAt(1);
3331  bool is_div = instruction->IsDiv();
3332
3333  switch (instruction->GetResultType()) {
3334    case Primitive::kPrimInt: {
3335      DCHECK_EQ(EAX, first.AsRegister<Register>());
3336      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
3337
3338      if (second.IsConstant()) {
3339        int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
3340
3341        if (imm == 0) {
3342          // Do not generate anything for 0. DivZeroCheck would forbid any generated code.
3343        } else if (imm == 1 || imm == -1) {
3344          DivRemOneOrMinusOne(instruction);
3345        } else if (is_div && IsPowerOfTwo(AbsOrMin(imm))) {
3346          DivByPowerOfTwo(instruction->AsDiv());
3347        } else {
3348          DCHECK(imm <= -2 || imm >= 2);
3349          GenerateDivRemWithAnyConstant(instruction);
3350        }
3351      } else {
3352        SlowPathCode* slow_path =
3353          new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(out.AsRegister<Register>(),
3354              is_div);
3355        codegen_->AddSlowPath(slow_path);
3356
3357        Register second_reg = second.AsRegister<Register>();
3358        // 0x80000000/-1 triggers an arithmetic exception!
3359        // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
3360        // it's safe to just use negl instead of more complex comparisons.
3361
3362        __ cmpl(second_reg, Immediate(-1));
3363        __ j(kEqual, slow_path->GetEntryLabel());
3364
3365        // edx:eax <- sign-extended of eax
3366        __ cdq();
3367        // eax = quotient, edx = remainder
3368        __ idivl(second_reg);
3369        __ Bind(slow_path->GetExitLabel());
3370      }
3371      break;
3372    }
3373
3374    case Primitive::kPrimLong: {
3375      InvokeRuntimeCallingConvention calling_convention;
3376      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
3377      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
3378      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
3379      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
3380      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
3381      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
3382
3383      if (is_div) {
3384        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLdiv),
3385                                instruction,
3386                                instruction->GetDexPc(),
3387                                nullptr);
3388        CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>();
3389      } else {
3390        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLmod),
3391                                instruction,
3392                                instruction->GetDexPc(),
3393                                nullptr);
3394        CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>();
3395      }
3396      break;
3397    }
3398
3399    default:
3400      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
3401  }
3402}
3403
3404void LocationsBuilderX86::VisitDiv(HDiv* div) {
3405  LocationSummary::CallKind call_kind = (div->GetResultType() == Primitive::kPrimLong)
3406      ? LocationSummary::kCall
3407      : LocationSummary::kNoCall;
3408  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
3409
3410  switch (div->GetResultType()) {
3411    case Primitive::kPrimInt: {
3412      locations->SetInAt(0, Location::RegisterLocation(EAX));
3413      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
3414      locations->SetOut(Location::SameAsFirstInput());
3415      // Intel uses edx:eax as the dividend.
3416      locations->AddTemp(Location::RegisterLocation(EDX));
3417      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
3418      // which enforces results to be in EAX and EDX, things are simpler if we use EAX also as
3419      // output and request another temp.
3420      if (div->InputAt(1)->IsIntConstant()) {
3421        locations->AddTemp(Location::RequiresRegister());
3422      }
3423      break;
3424    }
3425    case Primitive::kPrimLong: {
3426      InvokeRuntimeCallingConvention calling_convention;
3427      locations->SetInAt(0, Location::RegisterPairLocation(
3428          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
3429      locations->SetInAt(1, Location::RegisterPairLocation(
3430          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
3431      // Runtime helper puts the result in EAX, EDX.
3432      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
3433      break;
3434    }
3435    case Primitive::kPrimFloat:
3436    case Primitive::kPrimDouble: {
3437      locations->SetInAt(0, Location::RequiresFpuRegister());
3438      locations->SetInAt(1, Location::Any());
3439      locations->SetOut(Location::SameAsFirstInput());
3440      break;
3441    }
3442
3443    default:
3444      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
3445  }
3446}
3447
3448void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
3449  LocationSummary* locations = div->GetLocations();
3450  Location first = locations->InAt(0);
3451  Location second = locations->InAt(1);
3452
3453  switch (div->GetResultType()) {
3454    case Primitive::kPrimInt:
3455    case Primitive::kPrimLong: {
3456      GenerateDivRemIntegral(div);
3457      break;
3458    }
3459
3460    case Primitive::kPrimFloat: {
3461      if (second.IsFpuRegister()) {
3462        __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3463      } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3464        HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3465        DCHECK(!const_area->NeedsMaterialization());
3466        __ divss(first.AsFpuRegister<XmmRegister>(),
3467                 codegen_->LiteralFloatAddress(
3468                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
3469                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3470      } else {
3471        DCHECK(second.IsStackSlot());
3472        __ divss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3473      }
3474      break;
3475    }
3476
3477    case Primitive::kPrimDouble: {
3478      if (second.IsFpuRegister()) {
3479        __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3480      } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3481        HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3482        DCHECK(!const_area->NeedsMaterialization());
3483        __ divsd(first.AsFpuRegister<XmmRegister>(),
3484                 codegen_->LiteralDoubleAddress(
3485                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
3486                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3487      } else {
3488        DCHECK(second.IsDoubleStackSlot());
3489        __ divsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3490      }
3491      break;
3492    }
3493
3494    default:
3495      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
3496  }
3497}
3498
3499void LocationsBuilderX86::VisitRem(HRem* rem) {
3500  Primitive::Type type = rem->GetResultType();
3501
3502  LocationSummary::CallKind call_kind = (rem->GetResultType() == Primitive::kPrimLong)
3503      ? LocationSummary::kCall
3504      : LocationSummary::kNoCall;
3505  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
3506
3507  switch (type) {
3508    case Primitive::kPrimInt: {
3509      locations->SetInAt(0, Location::RegisterLocation(EAX));
3510      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
3511      locations->SetOut(Location::RegisterLocation(EDX));
3512      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
3513      // which enforces results to be in EAX and EDX, things are simpler if we use EDX also as
3514      // output and request another temp.
3515      if (rem->InputAt(1)->IsIntConstant()) {
3516        locations->AddTemp(Location::RequiresRegister());
3517      }
3518      break;
3519    }
3520    case Primitive::kPrimLong: {
3521      InvokeRuntimeCallingConvention calling_convention;
3522      locations->SetInAt(0, Location::RegisterPairLocation(
3523          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
3524      locations->SetInAt(1, Location::RegisterPairLocation(
3525          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
3526      // Runtime helper puts the result in EAX, EDX.
3527      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
3528      break;
3529    }
3530    case Primitive::kPrimDouble:
3531    case Primitive::kPrimFloat: {
3532      locations->SetInAt(0, Location::Any());
3533      locations->SetInAt(1, Location::Any());
3534      locations->SetOut(Location::RequiresFpuRegister());
3535      locations->AddTemp(Location::RegisterLocation(EAX));
3536      break;
3537    }
3538
3539    default:
3540      LOG(FATAL) << "Unexpected rem type " << type;
3541  }
3542}
3543
3544void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
3545  Primitive::Type type = rem->GetResultType();
3546  switch (type) {
3547    case Primitive::kPrimInt:
3548    case Primitive::kPrimLong: {
3549      GenerateDivRemIntegral(rem);
3550      break;
3551    }
3552    case Primitive::kPrimFloat:
3553    case Primitive::kPrimDouble: {
3554      GenerateRemFP(rem);
3555      break;
3556    }
3557    default:
3558      LOG(FATAL) << "Unexpected rem type " << type;
3559  }
3560}
3561
3562void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3563  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
3564      ? LocationSummary::kCallOnSlowPath
3565      : LocationSummary::kNoCall;
3566  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3567  switch (instruction->GetType()) {
3568    case Primitive::kPrimByte:
3569    case Primitive::kPrimChar:
3570    case Primitive::kPrimShort:
3571    case Primitive::kPrimInt: {
3572      locations->SetInAt(0, Location::Any());
3573      break;
3574    }
3575    case Primitive::kPrimLong: {
3576      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3577      if (!instruction->IsConstant()) {
3578        locations->AddTemp(Location::RequiresRegister());
3579      }
3580      break;
3581    }
3582    default:
3583      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
3584  }
3585  if (instruction->HasUses()) {
3586    locations->SetOut(Location::SameAsFirstInput());
3587  }
3588}
3589
3590void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3591  SlowPathCode* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
3592  codegen_->AddSlowPath(slow_path);
3593
3594  LocationSummary* locations = instruction->GetLocations();
3595  Location value = locations->InAt(0);
3596
3597  switch (instruction->GetType()) {
3598    case Primitive::kPrimByte:
3599    case Primitive::kPrimChar:
3600    case Primitive::kPrimShort:
3601    case Primitive::kPrimInt: {
3602      if (value.IsRegister()) {
3603        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
3604        __ j(kEqual, slow_path->GetEntryLabel());
3605      } else if (value.IsStackSlot()) {
3606        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
3607        __ j(kEqual, slow_path->GetEntryLabel());
3608      } else {
3609        DCHECK(value.IsConstant()) << value;
3610        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
3611        __ jmp(slow_path->GetEntryLabel());
3612        }
3613      }
3614      break;
3615    }
3616    case Primitive::kPrimLong: {
3617      if (value.IsRegisterPair()) {
3618        Register temp = locations->GetTemp(0).AsRegister<Register>();
3619        __ movl(temp, value.AsRegisterPairLow<Register>());
3620        __ orl(temp, value.AsRegisterPairHigh<Register>());
3621        __ j(kEqual, slow_path->GetEntryLabel());
3622      } else {
3623        DCHECK(value.IsConstant()) << value;
3624        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
3625          __ jmp(slow_path->GetEntryLabel());
3626        }
3627      }
3628      break;
3629    }
3630    default:
3631      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
3632  }
3633}
3634
3635void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
3636  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3637
3638  LocationSummary* locations =
3639      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
3640
3641  switch (op->GetResultType()) {
3642    case Primitive::kPrimInt:
3643    case Primitive::kPrimLong: {
3644      // Can't have Location::Any() and output SameAsFirstInput()
3645      locations->SetInAt(0, Location::RequiresRegister());
3646      // The shift count needs to be in CL or a constant.
3647      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
3648      locations->SetOut(Location::SameAsFirstInput());
3649      break;
3650    }
3651    default:
3652      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3653  }
3654}
3655
3656void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
3657  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3658
3659  LocationSummary* locations = op->GetLocations();
3660  Location first = locations->InAt(0);
3661  Location second = locations->InAt(1);
3662  DCHECK(first.Equals(locations->Out()));
3663
3664  switch (op->GetResultType()) {
3665    case Primitive::kPrimInt: {
3666      DCHECK(first.IsRegister());
3667      Register first_reg = first.AsRegister<Register>();
3668      if (second.IsRegister()) {
3669        Register second_reg = second.AsRegister<Register>();
3670        DCHECK_EQ(ECX, second_reg);
3671        if (op->IsShl()) {
3672          __ shll(first_reg, second_reg);
3673        } else if (op->IsShr()) {
3674          __ sarl(first_reg, second_reg);
3675        } else {
3676          __ shrl(first_reg, second_reg);
3677        }
3678      } else {
3679        int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue;
3680        if (shift == 0) {
3681          return;
3682        }
3683        Immediate imm(shift);
3684        if (op->IsShl()) {
3685          __ shll(first_reg, imm);
3686        } else if (op->IsShr()) {
3687          __ sarl(first_reg, imm);
3688        } else {
3689          __ shrl(first_reg, imm);
3690        }
3691      }
3692      break;
3693    }
3694    case Primitive::kPrimLong: {
3695      if (second.IsRegister()) {
3696        Register second_reg = second.AsRegister<Register>();
3697        DCHECK_EQ(ECX, second_reg);
3698        if (op->IsShl()) {
3699          GenerateShlLong(first, second_reg);
3700        } else if (op->IsShr()) {
3701          GenerateShrLong(first, second_reg);
3702        } else {
3703          GenerateUShrLong(first, second_reg);
3704        }
3705      } else {
3706        // Shift by a constant.
3707        int shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue;
3708        // Nothing to do if the shift is 0, as the input is already the output.
3709        if (shift != 0) {
3710          if (op->IsShl()) {
3711            GenerateShlLong(first, shift);
3712          } else if (op->IsShr()) {
3713            GenerateShrLong(first, shift);
3714          } else {
3715            GenerateUShrLong(first, shift);
3716          }
3717        }
3718      }
3719      break;
3720    }
3721    default:
3722      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3723  }
3724}
3725
3726void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, int shift) {
3727  Register low = loc.AsRegisterPairLow<Register>();
3728  Register high = loc.AsRegisterPairHigh<Register>();
3729  if (shift == 1) {
3730    // This is just an addition.
3731    __ addl(low, low);
3732    __ adcl(high, high);
3733  } else if (shift == 32) {
3734    // Shift by 32 is easy. High gets low, and low gets 0.
3735    codegen_->EmitParallelMoves(
3736        loc.ToLow(),
3737        loc.ToHigh(),
3738        Primitive::kPrimInt,
3739        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3740        loc.ToLow(),
3741        Primitive::kPrimInt);
3742  } else if (shift > 32) {
3743    // Low part becomes 0.  High part is low part << (shift-32).
3744    __ movl(high, low);
3745    __ shll(high, Immediate(shift - 32));
3746    __ xorl(low, low);
3747  } else {
3748    // Between 1 and 31.
3749    __ shld(high, low, Immediate(shift));
3750    __ shll(low, Immediate(shift));
3751  }
3752}
3753
3754void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
3755  NearLabel done;
3756  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
3757  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
3758  __ testl(shifter, Immediate(32));
3759  __ j(kEqual, &done);
3760  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
3761  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
3762  __ Bind(&done);
3763}
3764
3765void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, int shift) {
3766  Register low = loc.AsRegisterPairLow<Register>();
3767  Register high = loc.AsRegisterPairHigh<Register>();
3768  if (shift == 32) {
3769    // Need to copy the sign.
3770    DCHECK_NE(low, high);
3771    __ movl(low, high);
3772    __ sarl(high, Immediate(31));
3773  } else if (shift > 32) {
3774    DCHECK_NE(low, high);
3775    // High part becomes sign. Low part is shifted by shift - 32.
3776    __ movl(low, high);
3777    __ sarl(high, Immediate(31));
3778    __ sarl(low, Immediate(shift - 32));
3779  } else {
3780    // Between 1 and 31.
3781    __ shrd(low, high, Immediate(shift));
3782    __ sarl(high, Immediate(shift));
3783  }
3784}
3785
3786void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
3787  NearLabel done;
3788  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3789  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
3790  __ testl(shifter, Immediate(32));
3791  __ j(kEqual, &done);
3792  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3793  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
3794  __ Bind(&done);
3795}
3796
3797void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, int shift) {
3798  Register low = loc.AsRegisterPairLow<Register>();
3799  Register high = loc.AsRegisterPairHigh<Register>();
3800  if (shift == 32) {
3801    // Shift by 32 is easy. Low gets high, and high gets 0.
3802    codegen_->EmitParallelMoves(
3803        loc.ToHigh(),
3804        loc.ToLow(),
3805        Primitive::kPrimInt,
3806        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3807        loc.ToHigh(),
3808        Primitive::kPrimInt);
3809  } else if (shift > 32) {
3810    // Low part is high >> (shift - 32). High part becomes 0.
3811    __ movl(low, high);
3812    __ shrl(low, Immediate(shift - 32));
3813    __ xorl(high, high);
3814  } else {
3815    // Between 1 and 31.
3816    __ shrd(low, high, Immediate(shift));
3817    __ shrl(high, Immediate(shift));
3818  }
3819}
3820
3821void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
3822  NearLabel done;
3823  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3824  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
3825  __ testl(shifter, Immediate(32));
3826  __ j(kEqual, &done);
3827  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3828  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
3829  __ Bind(&done);
3830}
3831
3832void LocationsBuilderX86::VisitRor(HRor* ror) {
3833  LocationSummary* locations =
3834      new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall);
3835
3836  switch (ror->GetResultType()) {
3837    case Primitive::kPrimLong:
3838      // Add the temporary needed.
3839      locations->AddTemp(Location::RequiresRegister());
3840      FALLTHROUGH_INTENDED;
3841    case Primitive::kPrimInt:
3842      locations->SetInAt(0, Location::RequiresRegister());
3843      // The shift count needs to be in CL (unless it is a constant).
3844      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, ror->InputAt(1)));
3845      locations->SetOut(Location::SameAsFirstInput());
3846      break;
3847    default:
3848      LOG(FATAL) << "Unexpected operation type " << ror->GetResultType();
3849      UNREACHABLE();
3850  }
3851}
3852
3853void InstructionCodeGeneratorX86::VisitRor(HRor* ror) {
3854  LocationSummary* locations = ror->GetLocations();
3855  Location first = locations->InAt(0);
3856  Location second = locations->InAt(1);
3857
3858  if (ror->GetResultType() == Primitive::kPrimInt) {
3859    Register first_reg = first.AsRegister<Register>();
3860    if (second.IsRegister()) {
3861      Register second_reg = second.AsRegister<Register>();
3862      __ rorl(first_reg, second_reg);
3863    } else {
3864      Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue);
3865      __ rorl(first_reg, imm);
3866    }
3867    return;
3868  }
3869
3870  DCHECK_EQ(ror->GetResultType(), Primitive::kPrimLong);
3871  Register first_reg_lo = first.AsRegisterPairLow<Register>();
3872  Register first_reg_hi = first.AsRegisterPairHigh<Register>();
3873  Register temp_reg = locations->GetTemp(0).AsRegister<Register>();
3874  if (second.IsRegister()) {
3875    Register second_reg = second.AsRegister<Register>();
3876    DCHECK_EQ(second_reg, ECX);
3877    __ movl(temp_reg, first_reg_hi);
3878    __ shrd(first_reg_hi, first_reg_lo, second_reg);
3879    __ shrd(first_reg_lo, temp_reg, second_reg);
3880    __ movl(temp_reg, first_reg_hi);
3881    __ testl(second_reg, Immediate(32));
3882    __ cmovl(kNotEqual, first_reg_hi, first_reg_lo);
3883    __ cmovl(kNotEqual, first_reg_lo, temp_reg);
3884  } else {
3885    int32_t shift_amt =
3886      CodeGenerator::GetInt64ValueOf(second.GetConstant()) & kMaxLongShiftValue;
3887    if (shift_amt == 0) {
3888      // Already fine.
3889      return;
3890    }
3891    if (shift_amt == 32) {
3892      // Just swap.
3893      __ movl(temp_reg, first_reg_lo);
3894      __ movl(first_reg_lo, first_reg_hi);
3895      __ movl(first_reg_hi, temp_reg);
3896      return;
3897    }
3898
3899    Immediate imm(shift_amt);
3900    // Save the constents of the low value.
3901    __ movl(temp_reg, first_reg_lo);
3902
3903    // Shift right into low, feeding bits from high.
3904    __ shrd(first_reg_lo, first_reg_hi, imm);
3905
3906    // Shift right into high, feeding bits from the original low.
3907    __ shrd(first_reg_hi, temp_reg, imm);
3908
3909    // Swap if needed.
3910    if (shift_amt > 32) {
3911      __ movl(temp_reg, first_reg_lo);
3912      __ movl(first_reg_lo, first_reg_hi);
3913      __ movl(first_reg_hi, temp_reg);
3914    }
3915  }
3916}
3917
3918void LocationsBuilderX86::VisitShl(HShl* shl) {
3919  HandleShift(shl);
3920}
3921
3922void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
3923  HandleShift(shl);
3924}
3925
3926void LocationsBuilderX86::VisitShr(HShr* shr) {
3927  HandleShift(shr);
3928}
3929
3930void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
3931  HandleShift(shr);
3932}
3933
3934void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
3935  HandleShift(ushr);
3936}
3937
3938void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
3939  HandleShift(ushr);
3940}
3941
3942void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
3943  LocationSummary* locations =
3944      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3945  locations->SetOut(Location::RegisterLocation(EAX));
3946  if (instruction->IsStringAlloc()) {
3947    locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument));
3948  } else {
3949    InvokeRuntimeCallingConvention calling_convention;
3950    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3951    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3952  }
3953}
3954
3955void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
3956  // Note: if heap poisoning is enabled, the entry point takes cares
3957  // of poisoning the reference.
3958  if (instruction->IsStringAlloc()) {
3959    // String is allocated through StringFactory. Call NewEmptyString entry point.
3960    Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>();
3961    MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize);
3962    __ fs()->movl(temp, Address::Absolute(QUICK_ENTRY_POINT(pNewEmptyString)));
3963    __ call(Address(temp, code_offset.Int32Value()));
3964    codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3965  } else {
3966    codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3967                            instruction,
3968                            instruction->GetDexPc(),
3969                            nullptr);
3970    CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
3971    DCHECK(!codegen_->IsLeafMethod());
3972  }
3973}
3974
3975void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
3976  LocationSummary* locations =
3977      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3978  locations->SetOut(Location::RegisterLocation(EAX));
3979  InvokeRuntimeCallingConvention calling_convention;
3980  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3981  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3982  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3983}
3984
3985void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
3986  InvokeRuntimeCallingConvention calling_convention;
3987  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
3988  // Note: if heap poisoning is enabled, the entry point takes cares
3989  // of poisoning the reference.
3990  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3991                          instruction,
3992                          instruction->GetDexPc(),
3993                          nullptr);
3994  CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>();
3995  DCHECK(!codegen_->IsLeafMethod());
3996}
3997
3998void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
3999  LocationSummary* locations =
4000      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4001  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
4002  if (location.IsStackSlot()) {
4003    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
4004  } else if (location.IsDoubleStackSlot()) {
4005    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
4006  }
4007  locations->SetOut(location);
4008}
4009
4010void InstructionCodeGeneratorX86::VisitParameterValue(
4011    HParameterValue* instruction ATTRIBUTE_UNUSED) {
4012}
4013
4014void LocationsBuilderX86::VisitCurrentMethod(HCurrentMethod* instruction) {
4015  LocationSummary* locations =
4016      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4017  locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument));
4018}
4019
4020void InstructionCodeGeneratorX86::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
4021}
4022
4023void LocationsBuilderX86::VisitNot(HNot* not_) {
4024  LocationSummary* locations =
4025      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
4026  locations->SetInAt(0, Location::RequiresRegister());
4027  locations->SetOut(Location::SameAsFirstInput());
4028}
4029
4030void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
4031  LocationSummary* locations = not_->GetLocations();
4032  Location in = locations->InAt(0);
4033  Location out = locations->Out();
4034  DCHECK(in.Equals(out));
4035  switch (not_->GetResultType()) {
4036    case Primitive::kPrimInt:
4037      __ notl(out.AsRegister<Register>());
4038      break;
4039
4040    case Primitive::kPrimLong:
4041      __ notl(out.AsRegisterPairLow<Register>());
4042      __ notl(out.AsRegisterPairHigh<Register>());
4043      break;
4044
4045    default:
4046      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
4047  }
4048}
4049
4050void LocationsBuilderX86::VisitBooleanNot(HBooleanNot* bool_not) {
4051  LocationSummary* locations =
4052      new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
4053  locations->SetInAt(0, Location::RequiresRegister());
4054  locations->SetOut(Location::SameAsFirstInput());
4055}
4056
4057void InstructionCodeGeneratorX86::VisitBooleanNot(HBooleanNot* bool_not) {
4058  LocationSummary* locations = bool_not->GetLocations();
4059  Location in = locations->InAt(0);
4060  Location out = locations->Out();
4061  DCHECK(in.Equals(out));
4062  __ xorl(out.AsRegister<Register>(), Immediate(1));
4063}
4064
4065void LocationsBuilderX86::VisitCompare(HCompare* compare) {
4066  LocationSummary* locations =
4067      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
4068  switch (compare->InputAt(0)->GetType()) {
4069    case Primitive::kPrimLong: {
4070      locations->SetInAt(0, Location::RequiresRegister());
4071      locations->SetInAt(1, Location::Any());
4072      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
4073      break;
4074    }
4075    case Primitive::kPrimFloat:
4076    case Primitive::kPrimDouble: {
4077      locations->SetInAt(0, Location::RequiresFpuRegister());
4078      locations->SetInAt(1, Location::RequiresFpuRegister());
4079      locations->SetOut(Location::RequiresRegister());
4080      break;
4081    }
4082    default:
4083      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
4084  }
4085}
4086
4087void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
4088  LocationSummary* locations = compare->GetLocations();
4089  Register out = locations->Out().AsRegister<Register>();
4090  Location left = locations->InAt(0);
4091  Location right = locations->InAt(1);
4092
4093  NearLabel less, greater, done;
4094  switch (compare->InputAt(0)->GetType()) {
4095    case Primitive::kPrimLong: {
4096      Register left_low = left.AsRegisterPairLow<Register>();
4097      Register left_high = left.AsRegisterPairHigh<Register>();
4098      int32_t val_low = 0;
4099      int32_t val_high = 0;
4100      bool right_is_const = false;
4101
4102      if (right.IsConstant()) {
4103        DCHECK(right.GetConstant()->IsLongConstant());
4104        right_is_const = true;
4105        int64_t val = right.GetConstant()->AsLongConstant()->GetValue();
4106        val_low = Low32Bits(val);
4107        val_high = High32Bits(val);
4108      }
4109
4110      if (right.IsRegisterPair()) {
4111        __ cmpl(left_high, right.AsRegisterPairHigh<Register>());
4112      } else if (right.IsDoubleStackSlot()) {
4113        __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
4114      } else {
4115        DCHECK(right_is_const) << right;
4116        if (val_high == 0) {
4117          __ testl(left_high, left_high);
4118        } else {
4119          __ cmpl(left_high, Immediate(val_high));
4120        }
4121      }
4122      __ j(kLess, &less);  // Signed compare.
4123      __ j(kGreater, &greater);  // Signed compare.
4124      if (right.IsRegisterPair()) {
4125        __ cmpl(left_low, right.AsRegisterPairLow<Register>());
4126      } else if (right.IsDoubleStackSlot()) {
4127        __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
4128      } else {
4129        DCHECK(right_is_const) << right;
4130        if (val_low == 0) {
4131          __ testl(left_low, left_low);
4132        } else {
4133          __ cmpl(left_low, Immediate(val_low));
4134        }
4135      }
4136      break;
4137    }
4138    case Primitive::kPrimFloat: {
4139      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
4140      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
4141      break;
4142    }
4143    case Primitive::kPrimDouble: {
4144      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
4145      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
4146      break;
4147    }
4148    default:
4149      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
4150  }
4151  __ movl(out, Immediate(0));
4152  __ j(kEqual, &done);
4153  __ j(kBelow, &less);  // kBelow is for CF (unsigned & floats).
4154
4155  __ Bind(&greater);
4156  __ movl(out, Immediate(1));
4157  __ jmp(&done);
4158
4159  __ Bind(&less);
4160  __ movl(out, Immediate(-1));
4161
4162  __ Bind(&done);
4163}
4164
4165void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
4166  LocationSummary* locations =
4167      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4168  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
4169    locations->SetInAt(i, Location::Any());
4170  }
4171  locations->SetOut(Location::Any());
4172}
4173
4174void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) {
4175  LOG(FATAL) << "Unreachable";
4176}
4177
4178void CodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
4179  /*
4180   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
4181   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
4182   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
4183   */
4184  switch (kind) {
4185    case MemBarrierKind::kAnyAny: {
4186      MemoryFence();
4187      break;
4188    }
4189    case MemBarrierKind::kAnyStore:
4190    case MemBarrierKind::kLoadAny:
4191    case MemBarrierKind::kStoreStore: {
4192      // nop
4193      break;
4194    }
4195    default:
4196      LOG(FATAL) << "Unexpected memory barrier " << kind;
4197  }
4198}
4199
4200HInvokeStaticOrDirect::DispatchInfo CodeGeneratorX86::GetSupportedInvokeStaticOrDirectDispatch(
4201      const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
4202      MethodReference target_method ATTRIBUTE_UNUSED) {
4203  switch (desired_dispatch_info.code_ptr_location) {
4204    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
4205    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
4206      // For direct code, we actually prefer to call via the code pointer from ArtMethod*.
4207      // (Though the direct CALL ptr16:32 is available for consideration).
4208      return HInvokeStaticOrDirect::DispatchInfo {
4209        desired_dispatch_info.method_load_kind,
4210        HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod,
4211        desired_dispatch_info.method_load_data,
4212        0u
4213      };
4214    default:
4215      return desired_dispatch_info;
4216  }
4217}
4218
4219Register CodeGeneratorX86::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke,
4220                                                                 Register temp) {
4221  DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u);
4222  Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4223  if (!invoke->GetLocations()->Intrinsified()) {
4224    return location.AsRegister<Register>();
4225  }
4226  // For intrinsics we allow any location, so it may be on the stack.
4227  if (!location.IsRegister()) {
4228    __ movl(temp, Address(ESP, location.GetStackIndex()));
4229    return temp;
4230  }
4231  // For register locations, check if the register was saved. If so, get it from the stack.
4232  // Note: There is a chance that the register was saved but not overwritten, so we could
4233  // save one load. However, since this is just an intrinsic slow path we prefer this
4234  // simple and more robust approach rather that trying to determine if that's the case.
4235  SlowPathCode* slow_path = GetCurrentSlowPath();
4236  DCHECK(slow_path != nullptr);  // For intrinsified invokes the call is emitted on the slow path.
4237  if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) {
4238    int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>());
4239    __ movl(temp, Address(ESP, stack_offset));
4240    return temp;
4241  }
4242  return location.AsRegister<Register>();
4243}
4244
4245void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) {
4246  Location callee_method = temp;  // For all kinds except kRecursive, callee will be in temp.
4247  switch (invoke->GetMethodLoadKind()) {
4248    case HInvokeStaticOrDirect::MethodLoadKind::kStringInit:
4249      // temp = thread->string_init_entrypoint
4250      __ fs()->movl(temp.AsRegister<Register>(), Address::Absolute(invoke->GetStringInitOffset()));
4251      break;
4252    case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
4253      callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4254      break;
4255    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
4256      __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress()));
4257      break;
4258    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
4259      __ movl(temp.AsRegister<Register>(), Immediate(0));  // Placeholder.
4260      method_patches_.emplace_back(invoke->GetTargetMethod());
4261      __ Bind(&method_patches_.back().label);  // Bind the label at the end of the "movl" insn.
4262      break;
4263    case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
4264      Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke,
4265                                                                temp.AsRegister<Register>());
4266      uint32_t offset = invoke->GetDexCacheArrayOffset();
4267      __ movl(temp.AsRegister<Register>(), Address(base_reg, kDummy32BitOffset));
4268      // Add the patch entry and bind its label at the end of the instruction.
4269      pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, offset);
4270      __ Bind(&pc_relative_dex_cache_patches_.back().label);
4271      break;
4272    }
4273    case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
4274      Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4275      Register method_reg;
4276      Register reg = temp.AsRegister<Register>();
4277      if (current_method.IsRegister()) {
4278        method_reg = current_method.AsRegister<Register>();
4279      } else {
4280        DCHECK(IsBaseline() || invoke->GetLocations()->Intrinsified());
4281        DCHECK(!current_method.IsValid());
4282        method_reg = reg;
4283        __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
4284      }
4285      // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_;
4286      __ movl(reg, Address(method_reg,
4287                           ArtMethod::DexCacheResolvedMethodsOffset(kX86PointerSize).Int32Value()));
4288      // temp = temp[index_in_cache]
4289      uint32_t index_in_cache = invoke->GetTargetMethod().dex_method_index;
4290      __ movl(reg, Address(reg, CodeGenerator::GetCachePointerOffset(index_in_cache)));
4291      break;
4292    }
4293  }
4294
4295  switch (invoke->GetCodePtrLocation()) {
4296    case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf:
4297      __ call(GetFrameEntryLabel());
4298      break;
4299    case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: {
4300      relative_call_patches_.emplace_back(invoke->GetTargetMethod());
4301      Label* label = &relative_call_patches_.back().label;
4302      __ call(label);  // Bind to the patch label, override at link time.
4303      __ Bind(label);  // Bind the label at the end of the "call" insn.
4304      break;
4305    }
4306    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
4307    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
4308      // Filtered out by GetSupportedInvokeStaticOrDirectDispatch().
4309      LOG(FATAL) << "Unsupported";
4310      UNREACHABLE();
4311    case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
4312      // (callee_method + offset_of_quick_compiled_code)()
4313      __ call(Address(callee_method.AsRegister<Register>(),
4314                      ArtMethod::EntryPointFromQuickCompiledCodeOffset(
4315                          kX86WordSize).Int32Value()));
4316      break;
4317  }
4318
4319  DCHECK(!IsLeafMethod());
4320}
4321
4322void CodeGeneratorX86::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) {
4323  Register temp = temp_in.AsRegister<Register>();
4324  uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
4325      invoke->GetVTableIndex(), kX86PointerSize).Uint32Value();
4326
4327  // Use the calling convention instead of the location of the receiver, as
4328  // intrinsics may have put the receiver in a different register. In the intrinsics
4329  // slow path, the arguments have been moved to the right place, so here we are
4330  // guaranteed that the receiver is the first register of the calling convention.
4331  InvokeDexCallingConvention calling_convention;
4332  Register receiver = calling_convention.GetRegisterAt(0);
4333  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4334  // /* HeapReference<Class> */ temp = receiver->klass_
4335  __ movl(temp, Address(receiver, class_offset));
4336  MaybeRecordImplicitNullCheck(invoke);
4337  // Instead of simply (possibly) unpoisoning `temp` here, we should
4338  // emit a read barrier for the previous class reference load.
4339  // However this is not required in practice, as this is an
4340  // intermediate/temporary reference and because the current
4341  // concurrent copying collector keeps the from-space memory
4342  // intact/accessible until the end of the marking phase (the
4343  // concurrent copying collector may not in the future).
4344  __ MaybeUnpoisonHeapReference(temp);
4345  // temp = temp->GetMethodAt(method_offset);
4346  __ movl(temp, Address(temp, method_offset));
4347  // call temp->GetEntryPoint();
4348  __ call(Address(
4349      temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
4350}
4351
4352void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
4353  DCHECK(linker_patches->empty());
4354  size_t size =
4355      method_patches_.size() +
4356      relative_call_patches_.size() +
4357      pc_relative_dex_cache_patches_.size();
4358  linker_patches->reserve(size);
4359  // The label points to the end of the "movl" insn but the literal offset for method
4360  // patch needs to point to the embedded constant which occupies the last 4 bytes.
4361  constexpr uint32_t kLabelPositionToLiteralOffsetAdjustment = 4u;
4362  for (const MethodPatchInfo<Label>& info : method_patches_) {
4363    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4364    linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset,
4365                                                       info.target_method.dex_file,
4366                                                       info.target_method.dex_method_index));
4367  }
4368  for (const MethodPatchInfo<Label>& info : relative_call_patches_) {
4369    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4370    linker_patches->push_back(LinkerPatch::RelativeCodePatch(literal_offset,
4371                                                             info.target_method.dex_file,
4372                                                             info.target_method.dex_method_index));
4373  }
4374  for (const PcRelativeDexCacheAccessInfo& info : pc_relative_dex_cache_patches_) {
4375    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4376    linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(literal_offset,
4377                                                              &info.target_dex_file,
4378                                                              GetMethodAddressOffset(),
4379                                                              info.element_offset));
4380  }
4381}
4382
4383void CodeGeneratorX86::MarkGCCard(Register temp,
4384                                  Register card,
4385                                  Register object,
4386                                  Register value,
4387                                  bool value_can_be_null) {
4388  NearLabel is_null;
4389  if (value_can_be_null) {
4390    __ testl(value, value);
4391    __ j(kEqual, &is_null);
4392  }
4393  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
4394  __ movl(temp, object);
4395  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
4396  __ movb(Address(temp, card, TIMES_1, 0),
4397          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
4398  if (value_can_be_null) {
4399    __ Bind(&is_null);
4400  }
4401}
4402
4403void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
4404  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
4405
4406  bool object_field_get_with_read_barrier =
4407      kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
4408  LocationSummary* locations =
4409      new (GetGraph()->GetArena()) LocationSummary(instruction,
4410                                                   kEmitCompilerReadBarrier ?
4411                                                       LocationSummary::kCallOnSlowPath :
4412                                                       LocationSummary::kNoCall);
4413  locations->SetInAt(0, Location::RequiresRegister());
4414
4415  if (Primitive::IsFloatingPointType(instruction->GetType())) {
4416    locations->SetOut(Location::RequiresFpuRegister());
4417  } else {
4418    // The output overlaps in case of long: we don't want the low move
4419    // to overwrite the object's location.  Likewise, in the case of
4420    // an object field get with read barriers enabled, we do not want
4421    // the move to overwrite the object's location, as we need it to emit
4422    // the read barrier.
4423    locations->SetOut(
4424        Location::RequiresRegister(),
4425        (object_field_get_with_read_barrier || instruction->GetType() == Primitive::kPrimLong) ?
4426            Location::kOutputOverlap :
4427            Location::kNoOutputOverlap);
4428  }
4429
4430  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
4431    // Long values can be loaded atomically into an XMM using movsd.
4432    // So we use an XMM register as a temp to achieve atomicity (first
4433    // load the temp into the XMM and then copy the XMM into the
4434    // output, 32 bits at a time).
4435    locations->AddTemp(Location::RequiresFpuRegister());
4436  } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) {
4437    // We need a temporary register for the read barrier marking slow
4438    // path in CodeGeneratorX86::GenerateFieldLoadWithBakerReadBarrier.
4439    locations->AddTemp(Location::RequiresRegister());
4440  }
4441}
4442
4443void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
4444                                                 const FieldInfo& field_info) {
4445  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
4446
4447  LocationSummary* locations = instruction->GetLocations();
4448  Location base_loc = locations->InAt(0);
4449  Register base = base_loc.AsRegister<Register>();
4450  Location out = locations->Out();
4451  bool is_volatile = field_info.IsVolatile();
4452  Primitive::Type field_type = field_info.GetFieldType();
4453  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
4454
4455  switch (field_type) {
4456    case Primitive::kPrimBoolean: {
4457      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
4458      break;
4459    }
4460
4461    case Primitive::kPrimByte: {
4462      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
4463      break;
4464    }
4465
4466    case Primitive::kPrimShort: {
4467      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
4468      break;
4469    }
4470
4471    case Primitive::kPrimChar: {
4472      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
4473      break;
4474    }
4475
4476    case Primitive::kPrimInt:
4477      __ movl(out.AsRegister<Register>(), Address(base, offset));
4478      break;
4479
4480    case Primitive::kPrimNot: {
4481      // /* HeapReference<Object> */ out = *(base + offset)
4482      if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
4483        Location temp_loc = locations->GetTemp(0);
4484        // Note that a potential implicit null check is handled in this
4485        // CodeGeneratorX86::GenerateFieldLoadWithBakerReadBarrier call.
4486        codegen_->GenerateFieldLoadWithBakerReadBarrier(
4487            instruction, out, base, offset, temp_loc, /* needs_null_check */ true);
4488        if (is_volatile) {
4489          codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4490        }
4491      } else {
4492        __ movl(out.AsRegister<Register>(), Address(base, offset));
4493        codegen_->MaybeRecordImplicitNullCheck(instruction);
4494        if (is_volatile) {
4495          codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4496        }
4497        // If read barriers are enabled, emit read barriers other than
4498        // Baker's using a slow path (and also unpoison the loaded
4499        // reference, if heap poisoning is enabled).
4500        codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset);
4501      }
4502      break;
4503    }
4504
4505    case Primitive::kPrimLong: {
4506      if (is_volatile) {
4507        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
4508        __ movsd(temp, Address(base, offset));
4509        codegen_->MaybeRecordImplicitNullCheck(instruction);
4510        __ movd(out.AsRegisterPairLow<Register>(), temp);
4511        __ psrlq(temp, Immediate(32));
4512        __ movd(out.AsRegisterPairHigh<Register>(), temp);
4513      } else {
4514        DCHECK_NE(base, out.AsRegisterPairLow<Register>());
4515        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
4516        codegen_->MaybeRecordImplicitNullCheck(instruction);
4517        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
4518      }
4519      break;
4520    }
4521
4522    case Primitive::kPrimFloat: {
4523      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
4524      break;
4525    }
4526
4527    case Primitive::kPrimDouble: {
4528      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
4529      break;
4530    }
4531
4532    case Primitive::kPrimVoid:
4533      LOG(FATAL) << "Unreachable type " << field_type;
4534      UNREACHABLE();
4535  }
4536
4537  if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimLong) {
4538    // Potential implicit null checks, in the case of reference or
4539    // long fields, are handled in the previous switch statement.
4540  } else {
4541    codegen_->MaybeRecordImplicitNullCheck(instruction);
4542  }
4543
4544  if (is_volatile) {
4545    if (field_type == Primitive::kPrimNot) {
4546      // Memory barriers, in the case of references, are also handled
4547      // in the previous switch statement.
4548    } else {
4549      codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4550    }
4551  }
4552}
4553
4554void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
4555  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
4556
4557  LocationSummary* locations =
4558      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4559  locations->SetInAt(0, Location::RequiresRegister());
4560  bool is_volatile = field_info.IsVolatile();
4561  Primitive::Type field_type = field_info.GetFieldType();
4562  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
4563    || (field_type == Primitive::kPrimByte);
4564
4565  // The register allocator does not support multiple
4566  // inputs that die at entry with one in a specific register.
4567  if (is_byte_type) {
4568    // Ensure the value is in a byte register.
4569    locations->SetInAt(1, Location::RegisterLocation(EAX));
4570  } else if (Primitive::IsFloatingPointType(field_type)) {
4571    if (is_volatile && field_type == Primitive::kPrimDouble) {
4572      // In order to satisfy the semantics of volatile, this must be a single instruction store.
4573      locations->SetInAt(1, Location::RequiresFpuRegister());
4574    } else {
4575      locations->SetInAt(1, Location::FpuRegisterOrConstant(instruction->InputAt(1)));
4576    }
4577  } else if (is_volatile && field_type == Primitive::kPrimLong) {
4578    // In order to satisfy the semantics of volatile, this must be a single instruction store.
4579    locations->SetInAt(1, Location::RequiresRegister());
4580
4581    // 64bits value can be atomically written to an address with movsd and an XMM register.
4582    // We need two XMM registers because there's no easier way to (bit) copy a register pair
4583    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
4584    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
4585    // isolated cases when we need this it isn't worth adding the extra complexity.
4586    locations->AddTemp(Location::RequiresFpuRegister());
4587    locations->AddTemp(Location::RequiresFpuRegister());
4588  } else {
4589    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4590
4591    if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
4592      // Temporary registers for the write barrier.
4593      locations->AddTemp(Location::RequiresRegister());  // May be used for reference poisoning too.
4594      // Ensure the card is in a byte register.
4595      locations->AddTemp(Location::RegisterLocation(ECX));
4596    }
4597  }
4598}
4599
4600void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
4601                                                 const FieldInfo& field_info,
4602                                                 bool value_can_be_null) {
4603  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
4604
4605  LocationSummary* locations = instruction->GetLocations();
4606  Register base = locations->InAt(0).AsRegister<Register>();
4607  Location value = locations->InAt(1);
4608  bool is_volatile = field_info.IsVolatile();
4609  Primitive::Type field_type = field_info.GetFieldType();
4610  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
4611  bool needs_write_barrier =
4612      CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1));
4613
4614  if (is_volatile) {
4615    codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
4616  }
4617
4618  bool maybe_record_implicit_null_check_done = false;
4619
4620  switch (field_type) {
4621    case Primitive::kPrimBoolean:
4622    case Primitive::kPrimByte: {
4623      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
4624      break;
4625    }
4626
4627    case Primitive::kPrimShort:
4628    case Primitive::kPrimChar: {
4629      if (value.IsConstant()) {
4630        int16_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4631        __ movw(Address(base, offset), Immediate(v));
4632      } else {
4633        __ movw(Address(base, offset), value.AsRegister<Register>());
4634      }
4635      break;
4636    }
4637
4638    case Primitive::kPrimInt:
4639    case Primitive::kPrimNot: {
4640      if (kPoisonHeapReferences && needs_write_barrier) {
4641        // Note that in the case where `value` is a null reference,
4642        // we do not enter this block, as the reference does not
4643        // need poisoning.
4644        DCHECK_EQ(field_type, Primitive::kPrimNot);
4645        Register temp = locations->GetTemp(0).AsRegister<Register>();
4646        __ movl(temp, value.AsRegister<Register>());
4647        __ PoisonHeapReference(temp);
4648        __ movl(Address(base, offset), temp);
4649      } else if (value.IsConstant()) {
4650        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4651        __ movl(Address(base, offset), Immediate(v));
4652      } else {
4653        __ movl(Address(base, offset), value.AsRegister<Register>());
4654      }
4655      break;
4656    }
4657
4658    case Primitive::kPrimLong: {
4659      if (is_volatile) {
4660        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
4661        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
4662        __ movd(temp1, value.AsRegisterPairLow<Register>());
4663        __ movd(temp2, value.AsRegisterPairHigh<Register>());
4664        __ punpckldq(temp1, temp2);
4665        __ movsd(Address(base, offset), temp1);
4666        codegen_->MaybeRecordImplicitNullCheck(instruction);
4667      } else if (value.IsConstant()) {
4668        int64_t v = CodeGenerator::GetInt64ValueOf(value.GetConstant());
4669        __ movl(Address(base, offset), Immediate(Low32Bits(v)));
4670        codegen_->MaybeRecordImplicitNullCheck(instruction);
4671        __ movl(Address(base, kX86WordSize + offset), Immediate(High32Bits(v)));
4672      } else {
4673        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
4674        codegen_->MaybeRecordImplicitNullCheck(instruction);
4675        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
4676      }
4677      maybe_record_implicit_null_check_done = true;
4678      break;
4679    }
4680
4681    case Primitive::kPrimFloat: {
4682      if (value.IsConstant()) {
4683        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4684        __ movl(Address(base, offset), Immediate(v));
4685      } else {
4686        __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
4687      }
4688      break;
4689    }
4690
4691    case Primitive::kPrimDouble: {
4692      if (value.IsConstant()) {
4693        int64_t v = CodeGenerator::GetInt64ValueOf(value.GetConstant());
4694        __ movl(Address(base, offset), Immediate(Low32Bits(v)));
4695        codegen_->MaybeRecordImplicitNullCheck(instruction);
4696        __ movl(Address(base, kX86WordSize + offset), Immediate(High32Bits(v)));
4697        maybe_record_implicit_null_check_done = true;
4698      } else {
4699        __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
4700      }
4701      break;
4702    }
4703
4704    case Primitive::kPrimVoid:
4705      LOG(FATAL) << "Unreachable type " << field_type;
4706      UNREACHABLE();
4707  }
4708
4709  if (!maybe_record_implicit_null_check_done) {
4710    codegen_->MaybeRecordImplicitNullCheck(instruction);
4711  }
4712
4713  if (needs_write_barrier) {
4714    Register temp = locations->GetTemp(0).AsRegister<Register>();
4715    Register card = locations->GetTemp(1).AsRegister<Register>();
4716    codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>(), value_can_be_null);
4717  }
4718
4719  if (is_volatile) {
4720    codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
4721  }
4722}
4723
4724void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4725  HandleFieldGet(instruction, instruction->GetFieldInfo());
4726}
4727
4728void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4729  HandleFieldGet(instruction, instruction->GetFieldInfo());
4730}
4731
4732void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4733  HandleFieldSet(instruction, instruction->GetFieldInfo());
4734}
4735
4736void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4737  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4738}
4739
4740void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4741  HandleFieldSet(instruction, instruction->GetFieldInfo());
4742}
4743
4744void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4745  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4746}
4747
4748void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4749  HandleFieldGet(instruction, instruction->GetFieldInfo());
4750}
4751
4752void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4753  HandleFieldGet(instruction, instruction->GetFieldInfo());
4754}
4755
4756void LocationsBuilderX86::VisitUnresolvedInstanceFieldGet(
4757    HUnresolvedInstanceFieldGet* instruction) {
4758  FieldAccessCallingConventionX86 calling_convention;
4759  codegen_->CreateUnresolvedFieldLocationSummary(
4760      instruction, instruction->GetFieldType(), calling_convention);
4761}
4762
4763void InstructionCodeGeneratorX86::VisitUnresolvedInstanceFieldGet(
4764    HUnresolvedInstanceFieldGet* instruction) {
4765  FieldAccessCallingConventionX86 calling_convention;
4766  codegen_->GenerateUnresolvedFieldAccess(instruction,
4767                                          instruction->GetFieldType(),
4768                                          instruction->GetFieldIndex(),
4769                                          instruction->GetDexPc(),
4770                                          calling_convention);
4771}
4772
4773void LocationsBuilderX86::VisitUnresolvedInstanceFieldSet(
4774    HUnresolvedInstanceFieldSet* instruction) {
4775  FieldAccessCallingConventionX86 calling_convention;
4776  codegen_->CreateUnresolvedFieldLocationSummary(
4777      instruction, instruction->GetFieldType(), calling_convention);
4778}
4779
4780void InstructionCodeGeneratorX86::VisitUnresolvedInstanceFieldSet(
4781    HUnresolvedInstanceFieldSet* instruction) {
4782  FieldAccessCallingConventionX86 calling_convention;
4783  codegen_->GenerateUnresolvedFieldAccess(instruction,
4784                                          instruction->GetFieldType(),
4785                                          instruction->GetFieldIndex(),
4786                                          instruction->GetDexPc(),
4787                                          calling_convention);
4788}
4789
4790void LocationsBuilderX86::VisitUnresolvedStaticFieldGet(
4791    HUnresolvedStaticFieldGet* instruction) {
4792  FieldAccessCallingConventionX86 calling_convention;
4793  codegen_->CreateUnresolvedFieldLocationSummary(
4794      instruction, instruction->GetFieldType(), calling_convention);
4795}
4796
4797void InstructionCodeGeneratorX86::VisitUnresolvedStaticFieldGet(
4798    HUnresolvedStaticFieldGet* instruction) {
4799  FieldAccessCallingConventionX86 calling_convention;
4800  codegen_->GenerateUnresolvedFieldAccess(instruction,
4801                                          instruction->GetFieldType(),
4802                                          instruction->GetFieldIndex(),
4803                                          instruction->GetDexPc(),
4804                                          calling_convention);
4805}
4806
4807void LocationsBuilderX86::VisitUnresolvedStaticFieldSet(
4808    HUnresolvedStaticFieldSet* instruction) {
4809  FieldAccessCallingConventionX86 calling_convention;
4810  codegen_->CreateUnresolvedFieldLocationSummary(
4811      instruction, instruction->GetFieldType(), calling_convention);
4812}
4813
4814void InstructionCodeGeneratorX86::VisitUnresolvedStaticFieldSet(
4815    HUnresolvedStaticFieldSet* instruction) {
4816  FieldAccessCallingConventionX86 calling_convention;
4817  codegen_->GenerateUnresolvedFieldAccess(instruction,
4818                                          instruction->GetFieldType(),
4819                                          instruction->GetFieldIndex(),
4820                                          instruction->GetDexPc(),
4821                                          calling_convention);
4822}
4823
4824void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
4825  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
4826      ? LocationSummary::kCallOnSlowPath
4827      : LocationSummary::kNoCall;
4828  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4829  Location loc = codegen_->IsImplicitNullCheckAllowed(instruction)
4830      ? Location::RequiresRegister()
4831      : Location::Any();
4832  locations->SetInAt(0, loc);
4833  if (instruction->HasUses()) {
4834    locations->SetOut(Location::SameAsFirstInput());
4835  }
4836}
4837
4838void InstructionCodeGeneratorX86::GenerateImplicitNullCheck(HNullCheck* instruction) {
4839  if (codegen_->CanMoveNullCheckToUser(instruction)) {
4840    return;
4841  }
4842  LocationSummary* locations = instruction->GetLocations();
4843  Location obj = locations->InAt(0);
4844
4845  __ testl(EAX, Address(obj.AsRegister<Register>(), 0));
4846  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4847}
4848
4849void InstructionCodeGeneratorX86::GenerateExplicitNullCheck(HNullCheck* instruction) {
4850  SlowPathCode* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
4851  codegen_->AddSlowPath(slow_path);
4852
4853  LocationSummary* locations = instruction->GetLocations();
4854  Location obj = locations->InAt(0);
4855
4856  if (obj.IsRegister()) {
4857    __ testl(obj.AsRegister<Register>(), obj.AsRegister<Register>());
4858  } else if (obj.IsStackSlot()) {
4859    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
4860  } else {
4861    DCHECK(obj.IsConstant()) << obj;
4862    DCHECK(obj.GetConstant()->IsNullConstant());
4863    __ jmp(slow_path->GetEntryLabel());
4864    return;
4865  }
4866  __ j(kEqual, slow_path->GetEntryLabel());
4867}
4868
4869void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
4870  if (codegen_->IsImplicitNullCheckAllowed(instruction)) {
4871    GenerateImplicitNullCheck(instruction);
4872  } else {
4873    GenerateExplicitNullCheck(instruction);
4874  }
4875}
4876
4877void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
4878  bool object_array_get_with_read_barrier =
4879      kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
4880  LocationSummary* locations =
4881      new (GetGraph()->GetArena()) LocationSummary(instruction,
4882                                                   object_array_get_with_read_barrier ?
4883                                                       LocationSummary::kCallOnSlowPath :
4884                                                       LocationSummary::kNoCall);
4885  locations->SetInAt(0, Location::RequiresRegister());
4886  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4887  if (Primitive::IsFloatingPointType(instruction->GetType())) {
4888    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
4889  } else {
4890    // The output overlaps in case of long: we don't want the low move
4891    // to overwrite the array's location.  Likewise, in the case of an
4892    // object array get with read barriers enabled, we do not want the
4893    // move to overwrite the array's location, as we need it to emit
4894    // the read barrier.
4895    locations->SetOut(
4896        Location::RequiresRegister(),
4897        (instruction->GetType() == Primitive::kPrimLong || object_array_get_with_read_barrier) ?
4898            Location::kOutputOverlap :
4899            Location::kNoOutputOverlap);
4900  }
4901  // We need a temporary register for the read barrier marking slow
4902  // path in CodeGeneratorX86::GenerateArrayLoadWithBakerReadBarrier.
4903  if (object_array_get_with_read_barrier && kUseBakerReadBarrier) {
4904    locations->AddTemp(Location::RequiresRegister());
4905  }
4906}
4907
4908void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
4909  LocationSummary* locations = instruction->GetLocations();
4910  Location obj_loc = locations->InAt(0);
4911  Register obj = obj_loc.AsRegister<Register>();
4912  Location index = locations->InAt(1);
4913  Location out_loc = locations->Out();
4914
4915  Primitive::Type type = instruction->GetType();
4916  switch (type) {
4917    case Primitive::kPrimBoolean: {
4918      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
4919      Register out = out_loc.AsRegister<Register>();
4920      if (index.IsConstant()) {
4921        __ movzxb(out, Address(obj,
4922            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
4923      } else {
4924        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
4925      }
4926      break;
4927    }
4928
4929    case Primitive::kPrimByte: {
4930      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
4931      Register out = out_loc.AsRegister<Register>();
4932      if (index.IsConstant()) {
4933        __ movsxb(out, Address(obj,
4934            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
4935      } else {
4936        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
4937      }
4938      break;
4939    }
4940
4941    case Primitive::kPrimShort: {
4942      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
4943      Register out = out_loc.AsRegister<Register>();
4944      if (index.IsConstant()) {
4945        __ movsxw(out, Address(obj,
4946            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
4947      } else {
4948        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
4949      }
4950      break;
4951    }
4952
4953    case Primitive::kPrimChar: {
4954      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
4955      Register out = out_loc.AsRegister<Register>();
4956      if (index.IsConstant()) {
4957        __ movzxw(out, Address(obj,
4958            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
4959      } else {
4960        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
4961      }
4962      break;
4963    }
4964
4965    case Primitive::kPrimInt: {
4966      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4967      Register out = out_loc.AsRegister<Register>();
4968      if (index.IsConstant()) {
4969        __ movl(out, Address(obj,
4970            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
4971      } else {
4972        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
4973      }
4974      break;
4975    }
4976
4977    case Primitive::kPrimNot: {
4978      static_assert(
4979          sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
4980          "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
4981      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4982      // /* HeapReference<Object> */ out =
4983      //     *(obj + data_offset + index * sizeof(HeapReference<Object>))
4984      if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
4985        Location temp = locations->GetTemp(0);
4986        // Note that a potential implicit null check is handled in this
4987        // CodeGeneratorX86::GenerateArrayLoadWithBakerReadBarrier call.
4988        codegen_->GenerateArrayLoadWithBakerReadBarrier(
4989            instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true);
4990      } else {
4991        Register out = out_loc.AsRegister<Register>();
4992        if (index.IsConstant()) {
4993          uint32_t offset =
4994              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4995          __ movl(out, Address(obj, offset));
4996          codegen_->MaybeRecordImplicitNullCheck(instruction);
4997          // If read barriers are enabled, emit read barriers other than
4998          // Baker's using a slow path (and also unpoison the loaded
4999          // reference, if heap poisoning is enabled).
5000          codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset);
5001        } else {
5002          __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
5003          codegen_->MaybeRecordImplicitNullCheck(instruction);
5004          // If read barriers are enabled, emit read barriers other than
5005          // Baker's using a slow path (and also unpoison the loaded
5006          // reference, if heap poisoning is enabled).
5007          codegen_->MaybeGenerateReadBarrierSlow(
5008              instruction, out_loc, out_loc, obj_loc, data_offset, index);
5009        }
5010      }
5011      break;
5012    }
5013
5014    case Primitive::kPrimLong: {
5015      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
5016      DCHECK_NE(obj, out_loc.AsRegisterPairLow<Register>());
5017      if (index.IsConstant()) {
5018        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
5019        __ movl(out_loc.AsRegisterPairLow<Register>(), Address(obj, offset));
5020        codegen_->MaybeRecordImplicitNullCheck(instruction);
5021        __ movl(out_loc.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
5022      } else {
5023        __ movl(out_loc.AsRegisterPairLow<Register>(),
5024                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
5025        codegen_->MaybeRecordImplicitNullCheck(instruction);
5026        __ movl(out_loc.AsRegisterPairHigh<Register>(),
5027                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
5028      }
5029      break;
5030    }
5031
5032    case Primitive::kPrimFloat: {
5033      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
5034      XmmRegister out = out_loc.AsFpuRegister<XmmRegister>();
5035      if (index.IsConstant()) {
5036        __ movss(out, Address(obj,
5037            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
5038      } else {
5039        __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
5040      }
5041      break;
5042    }
5043
5044    case Primitive::kPrimDouble: {
5045      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
5046      XmmRegister out = out_loc.AsFpuRegister<XmmRegister>();
5047      if (index.IsConstant()) {
5048        __ movsd(out, Address(obj,
5049            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
5050      } else {
5051        __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
5052      }
5053      break;
5054    }
5055
5056    case Primitive::kPrimVoid:
5057      LOG(FATAL) << "Unreachable type " << type;
5058      UNREACHABLE();
5059  }
5060
5061  if (type == Primitive::kPrimNot || type == Primitive::kPrimLong) {
5062    // Potential implicit null checks, in the case of reference or
5063    // long arrays, are handled in the previous switch statement.
5064  } else {
5065    codegen_->MaybeRecordImplicitNullCheck(instruction);
5066  }
5067}
5068
5069void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
5070  // This location builder might end up asking to up to four registers, which is
5071  // not currently possible for baseline. The situation in which we need four
5072  // registers cannot be met by baseline though, because it has not run any
5073  // optimization.
5074
5075  Primitive::Type value_type = instruction->GetComponentType();
5076
5077  bool needs_write_barrier =
5078      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
5079  bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
5080  bool object_array_set_with_read_barrier =
5081      kEmitCompilerReadBarrier && (value_type == Primitive::kPrimNot);
5082
5083  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
5084      instruction,
5085      (may_need_runtime_call_for_type_check || object_array_set_with_read_barrier) ?
5086          LocationSummary::kCallOnSlowPath :
5087          LocationSummary::kNoCall);
5088
5089  bool is_byte_type = (value_type == Primitive::kPrimBoolean)
5090      || (value_type == Primitive::kPrimByte);
5091  // We need the inputs to be different than the output in case of long operation.
5092  // In case of a byte operation, the register allocator does not support multiple
5093  // inputs that die at entry with one in a specific register.
5094  locations->SetInAt(0, Location::RequiresRegister());
5095  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
5096  if (is_byte_type) {
5097    // Ensure the value is in a byte register.
5098    locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
5099  } else if (Primitive::IsFloatingPointType(value_type)) {
5100    locations->SetInAt(2, Location::FpuRegisterOrConstant(instruction->InputAt(2)));
5101  } else {
5102    locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
5103  }
5104  if (needs_write_barrier) {
5105    // Temporary registers for the write barrier.
5106    locations->AddTemp(Location::RequiresRegister());  // Possibly used for ref. poisoning too.
5107    // Ensure the card is in a byte register.
5108    locations->AddTemp(Location::RegisterLocation(ECX));
5109  }
5110}
5111
5112void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
5113  LocationSummary* locations = instruction->GetLocations();
5114  Location array_loc = locations->InAt(0);
5115  Register array = array_loc.AsRegister<Register>();
5116  Location index = locations->InAt(1);
5117  Location value = locations->InAt(2);
5118  Primitive::Type value_type = instruction->GetComponentType();
5119  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5120  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
5121  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
5122  bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
5123  bool needs_write_barrier =
5124      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
5125
5126  switch (value_type) {
5127    case Primitive::kPrimBoolean:
5128    case Primitive::kPrimByte: {
5129      uint32_t offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
5130      Address address = index.IsConstant()
5131          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + offset)
5132          : Address(array, index.AsRegister<Register>(), TIMES_1, offset);
5133      if (value.IsRegister()) {
5134        __ movb(address, value.AsRegister<ByteRegister>());
5135      } else {
5136        __ movb(address, Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
5137      }
5138      codegen_->MaybeRecordImplicitNullCheck(instruction);
5139      break;
5140    }
5141
5142    case Primitive::kPrimShort:
5143    case Primitive::kPrimChar: {
5144      uint32_t offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
5145      Address address = index.IsConstant()
5146          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + offset)
5147          : Address(array, index.AsRegister<Register>(), TIMES_2, offset);
5148      if (value.IsRegister()) {
5149        __ movw(address, value.AsRegister<Register>());
5150      } else {
5151        __ movw(address, Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
5152      }
5153      codegen_->MaybeRecordImplicitNullCheck(instruction);
5154      break;
5155    }
5156
5157    case Primitive::kPrimNot: {
5158      uint32_t offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
5159      Address address = index.IsConstant()
5160          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5161          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5162
5163      if (!value.IsRegister()) {
5164        // Just setting null.
5165        DCHECK(instruction->InputAt(2)->IsNullConstant());
5166        DCHECK(value.IsConstant()) << value;
5167        __ movl(address, Immediate(0));
5168        codegen_->MaybeRecordImplicitNullCheck(instruction);
5169        DCHECK(!needs_write_barrier);
5170        DCHECK(!may_need_runtime_call_for_type_check);
5171        break;
5172      }
5173
5174      DCHECK(needs_write_barrier);
5175      Register register_value = value.AsRegister<Register>();
5176      NearLabel done, not_null, do_put;
5177      SlowPathCode* slow_path = nullptr;
5178      Register temp = locations->GetTemp(0).AsRegister<Register>();
5179      if (may_need_runtime_call_for_type_check) {
5180        slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathX86(instruction);
5181        codegen_->AddSlowPath(slow_path);
5182        if (instruction->GetValueCanBeNull()) {
5183          __ testl(register_value, register_value);
5184          __ j(kNotEqual, &not_null);
5185          __ movl(address, Immediate(0));
5186          codegen_->MaybeRecordImplicitNullCheck(instruction);
5187          __ jmp(&done);
5188          __ Bind(&not_null);
5189        }
5190
5191        if (kEmitCompilerReadBarrier) {
5192          // When read barriers are enabled, the type checking
5193          // instrumentation requires two read barriers:
5194          //
5195          //   __ movl(temp2, temp);
5196          //   // /* HeapReference<Class> */ temp = temp->component_type_
5197          //   __ movl(temp, Address(temp, component_offset));
5198          //   codegen_->GenerateReadBarrierSlow(
5199          //       instruction, temp_loc, temp_loc, temp2_loc, component_offset);
5200          //
5201          //   // /* HeapReference<Class> */ temp2 = register_value->klass_
5202          //   __ movl(temp2, Address(register_value, class_offset));
5203          //   codegen_->GenerateReadBarrierSlow(
5204          //       instruction, temp2_loc, temp2_loc, value, class_offset, temp_loc);
5205          //
5206          //   __ cmpl(temp, temp2);
5207          //
5208          // However, the second read barrier may trash `temp`, as it
5209          // is a temporary register, and as such would not be saved
5210          // along with live registers before calling the runtime (nor
5211          // restored afterwards).  So in this case, we bail out and
5212          // delegate the work to the array set slow path.
5213          //
5214          // TODO: Extend the register allocator to support a new
5215          // "(locally) live temp" location so as to avoid always
5216          // going into the slow path when read barriers are enabled.
5217          __ jmp(slow_path->GetEntryLabel());
5218        } else {
5219          // /* HeapReference<Class> */ temp = array->klass_
5220          __ movl(temp, Address(array, class_offset));
5221          codegen_->MaybeRecordImplicitNullCheck(instruction);
5222          __ MaybeUnpoisonHeapReference(temp);
5223
5224          // /* HeapReference<Class> */ temp = temp->component_type_
5225          __ movl(temp, Address(temp, component_offset));
5226          // If heap poisoning is enabled, no need to unpoison `temp`
5227          // nor the object reference in `register_value->klass`, as
5228          // we are comparing two poisoned references.
5229          __ cmpl(temp, Address(register_value, class_offset));
5230
5231          if (instruction->StaticTypeOfArrayIsObjectArray()) {
5232            __ j(kEqual, &do_put);
5233            // If heap poisoning is enabled, the `temp` reference has
5234            // not been unpoisoned yet; unpoison it now.
5235            __ MaybeUnpoisonHeapReference(temp);
5236
5237            // /* HeapReference<Class> */ temp = temp->super_class_
5238            __ movl(temp, Address(temp, super_offset));
5239            // If heap poisoning is enabled, no need to unpoison
5240            // `temp`, as we are comparing against null below.
5241            __ testl(temp, temp);
5242            __ j(kNotEqual, slow_path->GetEntryLabel());
5243            __ Bind(&do_put);
5244          } else {
5245            __ j(kNotEqual, slow_path->GetEntryLabel());
5246          }
5247        }
5248      }
5249
5250      if (kPoisonHeapReferences) {
5251        __ movl(temp, register_value);
5252        __ PoisonHeapReference(temp);
5253        __ movl(address, temp);
5254      } else {
5255        __ movl(address, register_value);
5256      }
5257      if (!may_need_runtime_call_for_type_check) {
5258        codegen_->MaybeRecordImplicitNullCheck(instruction);
5259      }
5260
5261      Register card = locations->GetTemp(1).AsRegister<Register>();
5262      codegen_->MarkGCCard(
5263          temp, card, array, value.AsRegister<Register>(), instruction->GetValueCanBeNull());
5264      __ Bind(&done);
5265
5266      if (slow_path != nullptr) {
5267        __ Bind(slow_path->GetExitLabel());
5268      }
5269
5270      break;
5271    }
5272
5273    case Primitive::kPrimInt: {
5274      uint32_t offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
5275      Address address = index.IsConstant()
5276          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5277          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5278      if (value.IsRegister()) {
5279        __ movl(address, value.AsRegister<Register>());
5280      } else {
5281        DCHECK(value.IsConstant()) << value;
5282        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
5283        __ movl(address, Immediate(v));
5284      }
5285      codegen_->MaybeRecordImplicitNullCheck(instruction);
5286      break;
5287    }
5288
5289    case Primitive::kPrimLong: {
5290      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
5291      if (index.IsConstant()) {
5292        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
5293        if (value.IsRegisterPair()) {
5294          __ movl(Address(array, offset), value.AsRegisterPairLow<Register>());
5295          codegen_->MaybeRecordImplicitNullCheck(instruction);
5296          __ movl(Address(array, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
5297        } else {
5298          DCHECK(value.IsConstant());
5299          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
5300          __ movl(Address(array, offset), Immediate(Low32Bits(val)));
5301          codegen_->MaybeRecordImplicitNullCheck(instruction);
5302          __ movl(Address(array, offset + kX86WordSize), Immediate(High32Bits(val)));
5303        }
5304      } else {
5305        if (value.IsRegisterPair()) {
5306          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset),
5307                  value.AsRegisterPairLow<Register>());
5308          codegen_->MaybeRecordImplicitNullCheck(instruction);
5309          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
5310                  value.AsRegisterPairHigh<Register>());
5311        } else {
5312          DCHECK(value.IsConstant());
5313          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
5314          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset),
5315                  Immediate(Low32Bits(val)));
5316          codegen_->MaybeRecordImplicitNullCheck(instruction);
5317          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
5318                  Immediate(High32Bits(val)));
5319        }
5320      }
5321      break;
5322    }
5323
5324    case Primitive::kPrimFloat: {
5325      uint32_t offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
5326      Address address = index.IsConstant()
5327          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5328          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5329      if (value.IsFpuRegister()) {
5330        __ movss(address, value.AsFpuRegister<XmmRegister>());
5331      } else {
5332        DCHECK(value.IsConstant());
5333        int32_t v = bit_cast<int32_t, float>(value.GetConstant()->AsFloatConstant()->GetValue());
5334        __ movl(address, Immediate(v));
5335      }
5336      codegen_->MaybeRecordImplicitNullCheck(instruction);
5337      break;
5338    }
5339
5340    case Primitive::kPrimDouble: {
5341      uint32_t offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
5342      Address address = index.IsConstant()
5343          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + offset)
5344          : Address(array, index.AsRegister<Register>(), TIMES_8, offset);
5345      if (value.IsFpuRegister()) {
5346        __ movsd(address, value.AsFpuRegister<XmmRegister>());
5347      } else {
5348        DCHECK(value.IsConstant());
5349        Address address_hi = index.IsConstant() ?
5350            Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) +
5351                           offset + kX86WordSize) :
5352            Address(array, index.AsRegister<Register>(), TIMES_8, offset + kX86WordSize);
5353        int64_t v = bit_cast<int64_t, double>(value.GetConstant()->AsDoubleConstant()->GetValue());
5354        __ movl(address, Immediate(Low32Bits(v)));
5355        codegen_->MaybeRecordImplicitNullCheck(instruction);
5356        __ movl(address_hi, Immediate(High32Bits(v)));
5357      }
5358      break;
5359    }
5360
5361    case Primitive::kPrimVoid:
5362      LOG(FATAL) << "Unreachable type " << instruction->GetType();
5363      UNREACHABLE();
5364  }
5365}
5366
5367void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
5368  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
5369  locations->SetInAt(0, Location::RequiresRegister());
5370  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
5371}
5372
5373void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
5374  LocationSummary* locations = instruction->GetLocations();
5375  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
5376  Register obj = locations->InAt(0).AsRegister<Register>();
5377  Register out = locations->Out().AsRegister<Register>();
5378  __ movl(out, Address(obj, offset));
5379  codegen_->MaybeRecordImplicitNullCheck(instruction);
5380}
5381
5382void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
5383  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
5384      ? LocationSummary::kCallOnSlowPath
5385      : LocationSummary::kNoCall;
5386  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5387  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
5388  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
5389  if (instruction->HasUses()) {
5390    locations->SetOut(Location::SameAsFirstInput());
5391  }
5392}
5393
5394void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
5395  LocationSummary* locations = instruction->GetLocations();
5396  Location index_loc = locations->InAt(0);
5397  Location length_loc = locations->InAt(1);
5398  SlowPathCode* slow_path =
5399    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(instruction);
5400
5401  if (length_loc.IsConstant()) {
5402    int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
5403    if (index_loc.IsConstant()) {
5404      // BCE will remove the bounds check if we are guarenteed to pass.
5405      int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
5406      if (index < 0 || index >= length) {
5407        codegen_->AddSlowPath(slow_path);
5408        __ jmp(slow_path->GetEntryLabel());
5409      } else {
5410        // Some optimization after BCE may have generated this, and we should not
5411        // generate a bounds check if it is a valid range.
5412      }
5413      return;
5414    }
5415
5416    // We have to reverse the jump condition because the length is the constant.
5417    Register index_reg = index_loc.AsRegister<Register>();
5418    __ cmpl(index_reg, Immediate(length));
5419    codegen_->AddSlowPath(slow_path);
5420    __ j(kAboveEqual, slow_path->GetEntryLabel());
5421  } else {
5422    Register length = length_loc.AsRegister<Register>();
5423    if (index_loc.IsConstant()) {
5424      int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
5425      __ cmpl(length, Immediate(value));
5426    } else {
5427      __ cmpl(length, index_loc.AsRegister<Register>());
5428    }
5429    codegen_->AddSlowPath(slow_path);
5430    __ j(kBelowEqual, slow_path->GetEntryLabel());
5431  }
5432}
5433
5434void LocationsBuilderX86::VisitTemporary(HTemporary* temp) {
5435  temp->SetLocations(nullptr);
5436}
5437
5438void InstructionCodeGeneratorX86::VisitTemporary(HTemporary* temp ATTRIBUTE_UNUSED) {
5439  // Nothing to do, this is driven by the code generator.
5440}
5441
5442void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) {
5443  LOG(FATAL) << "Unreachable";
5444}
5445
5446void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
5447  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
5448}
5449
5450void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
5451  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
5452}
5453
5454void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
5455  HBasicBlock* block = instruction->GetBlock();
5456  if (block->GetLoopInformation() != nullptr) {
5457    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
5458    // The back edge will generate the suspend check.
5459    return;
5460  }
5461  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
5462    // The goto will generate the suspend check.
5463    return;
5464  }
5465  GenerateSuspendCheck(instruction, nullptr);
5466}
5467
5468void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
5469                                                       HBasicBlock* successor) {
5470  SuspendCheckSlowPathX86* slow_path =
5471      down_cast<SuspendCheckSlowPathX86*>(instruction->GetSlowPath());
5472  if (slow_path == nullptr) {
5473    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
5474    instruction->SetSlowPath(slow_path);
5475    codegen_->AddSlowPath(slow_path);
5476    if (successor != nullptr) {
5477      DCHECK(successor->IsLoopHeader());
5478      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
5479    }
5480  } else {
5481    DCHECK_EQ(slow_path->GetSuccessor(), successor);
5482  }
5483
5484  __ fs()->cmpw(Address::Absolute(Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()),
5485                Immediate(0));
5486  if (successor == nullptr) {
5487    __ j(kNotEqual, slow_path->GetEntryLabel());
5488    __ Bind(slow_path->GetReturnLabel());
5489  } else {
5490    __ j(kEqual, codegen_->GetLabelOf(successor));
5491    __ jmp(slow_path->GetEntryLabel());
5492  }
5493}
5494
5495X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
5496  return codegen_->GetAssembler();
5497}
5498
5499void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) {
5500  ScratchRegisterScope ensure_scratch(
5501      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5502  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5503  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5504  __ movl(temp_reg, Address(ESP, src + stack_offset));
5505  __ movl(Address(ESP, dst + stack_offset), temp_reg);
5506}
5507
5508void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) {
5509  ScratchRegisterScope ensure_scratch(
5510      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5511  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5512  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5513  __ movl(temp_reg, Address(ESP, src + stack_offset));
5514  __ movl(Address(ESP, dst + stack_offset), temp_reg);
5515  __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize));
5516  __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg);
5517}
5518
5519void ParallelMoveResolverX86::EmitMove(size_t index) {
5520  MoveOperands* move = moves_[index];
5521  Location source = move->GetSource();
5522  Location destination = move->GetDestination();
5523
5524  if (source.IsRegister()) {
5525    if (destination.IsRegister()) {
5526      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5527    } else {
5528      DCHECK(destination.IsStackSlot());
5529      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
5530    }
5531  } else if (source.IsFpuRegister()) {
5532    if (destination.IsFpuRegister()) {
5533      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5534    } else if (destination.IsStackSlot()) {
5535      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
5536    } else {
5537      DCHECK(destination.IsDoubleStackSlot());
5538      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
5539    }
5540  } else if (source.IsStackSlot()) {
5541    if (destination.IsRegister()) {
5542      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
5543    } else if (destination.IsFpuRegister()) {
5544      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
5545    } else {
5546      DCHECK(destination.IsStackSlot());
5547      MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex());
5548    }
5549  } else if (source.IsDoubleStackSlot()) {
5550    if (destination.IsFpuRegister()) {
5551      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
5552    } else {
5553      DCHECK(destination.IsDoubleStackSlot()) << destination;
5554      MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex());
5555    }
5556  } else if (source.IsConstant()) {
5557    HConstant* constant = source.GetConstant();
5558    if (constant->IsIntConstant() || constant->IsNullConstant()) {
5559      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
5560      if (destination.IsRegister()) {
5561        if (value == 0) {
5562          __ xorl(destination.AsRegister<Register>(), destination.AsRegister<Register>());
5563        } else {
5564          __ movl(destination.AsRegister<Register>(), Immediate(value));
5565        }
5566      } else {
5567        DCHECK(destination.IsStackSlot()) << destination;
5568        __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
5569      }
5570    } else if (constant->IsFloatConstant()) {
5571      float fp_value = constant->AsFloatConstant()->GetValue();
5572      int32_t value = bit_cast<int32_t, float>(fp_value);
5573      Immediate imm(value);
5574      if (destination.IsFpuRegister()) {
5575        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
5576        if (value == 0) {
5577          // Easy handling of 0.0.
5578          __ xorps(dest, dest);
5579        } else {
5580          ScratchRegisterScope ensure_scratch(
5581              this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5582          Register temp = static_cast<Register>(ensure_scratch.GetRegister());
5583          __ movl(temp, Immediate(value));
5584          __ movd(dest, temp);
5585        }
5586      } else {
5587        DCHECK(destination.IsStackSlot()) << destination;
5588        __ movl(Address(ESP, destination.GetStackIndex()), imm);
5589      }
5590    } else if (constant->IsLongConstant()) {
5591      int64_t value = constant->AsLongConstant()->GetValue();
5592      int32_t low_value = Low32Bits(value);
5593      int32_t high_value = High32Bits(value);
5594      Immediate low(low_value);
5595      Immediate high(high_value);
5596      if (destination.IsDoubleStackSlot()) {
5597        __ movl(Address(ESP, destination.GetStackIndex()), low);
5598        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
5599      } else {
5600        __ movl(destination.AsRegisterPairLow<Register>(), low);
5601        __ movl(destination.AsRegisterPairHigh<Register>(), high);
5602      }
5603    } else {
5604      DCHECK(constant->IsDoubleConstant());
5605      double dbl_value = constant->AsDoubleConstant()->GetValue();
5606      int64_t value = bit_cast<int64_t, double>(dbl_value);
5607      int32_t low_value = Low32Bits(value);
5608      int32_t high_value = High32Bits(value);
5609      Immediate low(low_value);
5610      Immediate high(high_value);
5611      if (destination.IsFpuRegister()) {
5612        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
5613        if (value == 0) {
5614          // Easy handling of 0.0.
5615          __ xorpd(dest, dest);
5616        } else {
5617          __ pushl(high);
5618          __ pushl(low);
5619          __ movsd(dest, Address(ESP, 0));
5620          __ addl(ESP, Immediate(8));
5621        }
5622      } else {
5623        DCHECK(destination.IsDoubleStackSlot()) << destination;
5624        __ movl(Address(ESP, destination.GetStackIndex()), low);
5625        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
5626      }
5627    }
5628  } else {
5629    LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
5630  }
5631}
5632
5633void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
5634  Register suggested_scratch = reg == EAX ? EBX : EAX;
5635  ScratchRegisterScope ensure_scratch(
5636      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
5637
5638  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5639  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
5640  __ movl(Address(ESP, mem + stack_offset), reg);
5641  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
5642}
5643
5644void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) {
5645  ScratchRegisterScope ensure_scratch(
5646      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5647
5648  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5649  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5650  __ movl(temp_reg, Address(ESP, mem + stack_offset));
5651  __ movss(Address(ESP, mem + stack_offset), reg);
5652  __ movd(reg, temp_reg);
5653}
5654
5655void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
5656  ScratchRegisterScope ensure_scratch1(
5657      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5658
5659  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
5660  ScratchRegisterScope ensure_scratch2(
5661      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
5662
5663  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
5664  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
5665  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
5666  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
5667  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
5668  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
5669}
5670
5671void ParallelMoveResolverX86::EmitSwap(size_t index) {
5672  MoveOperands* move = moves_[index];
5673  Location source = move->GetSource();
5674  Location destination = move->GetDestination();
5675
5676  if (source.IsRegister() && destination.IsRegister()) {
5677    // Use XOR swap algorithm to avoid serializing XCHG instruction or using a temporary.
5678    DCHECK_NE(destination.AsRegister<Register>(), source.AsRegister<Register>());
5679    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5680    __ xorl(source.AsRegister<Register>(), destination.AsRegister<Register>());
5681    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5682  } else if (source.IsRegister() && destination.IsStackSlot()) {
5683    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
5684  } else if (source.IsStackSlot() && destination.IsRegister()) {
5685    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
5686  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
5687    Exchange(destination.GetStackIndex(), source.GetStackIndex());
5688  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
5689    // Use XOR Swap algorithm to avoid a temporary.
5690    DCHECK_NE(source.reg(), destination.reg());
5691    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5692    __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
5693    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5694  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
5695    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
5696  } else if (destination.IsFpuRegister() && source.IsStackSlot()) {
5697    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
5698  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
5699    // Take advantage of the 16 bytes in the XMM register.
5700    XmmRegister reg = source.AsFpuRegister<XmmRegister>();
5701    Address stack(ESP, destination.GetStackIndex());
5702    // Load the double into the high doubleword.
5703    __ movhpd(reg, stack);
5704
5705    // Store the low double into the destination.
5706    __ movsd(stack, reg);
5707
5708    // Move the high double to the low double.
5709    __ psrldq(reg, Immediate(8));
5710  } else if (destination.IsFpuRegister() && source.IsDoubleStackSlot()) {
5711    // Take advantage of the 16 bytes in the XMM register.
5712    XmmRegister reg = destination.AsFpuRegister<XmmRegister>();
5713    Address stack(ESP, source.GetStackIndex());
5714    // Load the double into the high doubleword.
5715    __ movhpd(reg, stack);
5716
5717    // Store the low double into the destination.
5718    __ movsd(stack, reg);
5719
5720    // Move the high double to the low double.
5721    __ psrldq(reg, Immediate(8));
5722  } else if (destination.IsDoubleStackSlot() && source.IsDoubleStackSlot()) {
5723    Exchange(destination.GetStackIndex(), source.GetStackIndex());
5724    Exchange(destination.GetHighStackIndex(kX86WordSize), source.GetHighStackIndex(kX86WordSize));
5725  } else {
5726    LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination;
5727  }
5728}
5729
5730void ParallelMoveResolverX86::SpillScratch(int reg) {
5731  __ pushl(static_cast<Register>(reg));
5732}
5733
5734void ParallelMoveResolverX86::RestoreScratch(int reg) {
5735  __ popl(static_cast<Register>(reg));
5736}
5737
5738void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
5739  InvokeRuntimeCallingConvention calling_convention;
5740  CodeGenerator::CreateLoadClassLocationSummary(
5741      cls,
5742      Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
5743      Location::RegisterLocation(EAX),
5744      /* code_generator_supports_read_barrier */ true);
5745}
5746
5747void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
5748  LocationSummary* locations = cls->GetLocations();
5749  if (cls->NeedsAccessCheck()) {
5750    codegen_->MoveConstant(locations->GetTemp(0), cls->GetTypeIndex());
5751    codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInitializeTypeAndVerifyAccess),
5752                            cls,
5753                            cls->GetDexPc(),
5754                            nullptr);
5755    CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>();
5756    return;
5757  }
5758
5759  Location out_loc = locations->Out();
5760  Register out = out_loc.AsRegister<Register>();
5761  Register current_method = locations->InAt(0).AsRegister<Register>();
5762
5763  if (cls->IsReferrersClass()) {
5764    DCHECK(!cls->CanCallRuntime());
5765    DCHECK(!cls->MustGenerateClinitCheck());
5766    // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
5767    GenerateGcRootFieldLoad(
5768        cls, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
5769  } else {
5770    // /* GcRoot<mirror::Class>[] */ out =
5771    //        current_method.ptr_sized_fields_->dex_cache_resolved_types_
5772    __ movl(out, Address(current_method,
5773                         ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value()));
5774    // /* GcRoot<mirror::Class> */ out = out[type_index]
5775    GenerateGcRootFieldLoad(cls, out_loc, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex()));
5776
5777    if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) {
5778      DCHECK(cls->CanCallRuntime());
5779      SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
5780          cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
5781      codegen_->AddSlowPath(slow_path);
5782
5783      if (!cls->IsInDexCache()) {
5784        __ testl(out, out);
5785        __ j(kEqual, slow_path->GetEntryLabel());
5786      }
5787
5788      if (cls->MustGenerateClinitCheck()) {
5789        GenerateClassInitializationCheck(slow_path, out);
5790      } else {
5791        __ Bind(slow_path->GetExitLabel());
5792      }
5793    }
5794  }
5795}
5796
5797void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
5798  LocationSummary* locations =
5799      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
5800  locations->SetInAt(0, Location::RequiresRegister());
5801  if (check->HasUses()) {
5802    locations->SetOut(Location::SameAsFirstInput());
5803  }
5804}
5805
5806void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
5807  // We assume the class to not be null.
5808  SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
5809      check->GetLoadClass(), check, check->GetDexPc(), true);
5810  codegen_->AddSlowPath(slow_path);
5811  GenerateClassInitializationCheck(slow_path,
5812                                   check->GetLocations()->InAt(0).AsRegister<Register>());
5813}
5814
5815void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
5816    SlowPathCode* slow_path, Register class_reg) {
5817  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
5818          Immediate(mirror::Class::kStatusInitialized));
5819  __ j(kLess, slow_path->GetEntryLabel());
5820  __ Bind(slow_path->GetExitLabel());
5821  // No need for memory fence, thanks to the X86 memory model.
5822}
5823
5824void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
5825  LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier)
5826      ? LocationSummary::kCallOnSlowPath
5827      : LocationSummary::kNoCall;
5828  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
5829  locations->SetInAt(0, Location::RequiresRegister());
5830  locations->SetOut(Location::RequiresRegister());
5831}
5832
5833void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
5834  LocationSummary* locations = load->GetLocations();
5835  Location out_loc = locations->Out();
5836  Register out = out_loc.AsRegister<Register>();
5837  Register current_method = locations->InAt(0).AsRegister<Register>();
5838
5839  // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
5840  GenerateGcRootFieldLoad(
5841      load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
5842  // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
5843  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
5844  // /* GcRoot<mirror::String> */ out = out[string_index]
5845  GenerateGcRootFieldLoad(
5846      load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex()));
5847
5848  if (!load->IsInDexCache()) {
5849    SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
5850    codegen_->AddSlowPath(slow_path);
5851    __ testl(out, out);
5852    __ j(kEqual, slow_path->GetEntryLabel());
5853    __ Bind(slow_path->GetExitLabel());
5854  }
5855}
5856
5857static Address GetExceptionTlsAddress() {
5858  return Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
5859}
5860
5861void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
5862  LocationSummary* locations =
5863      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
5864  locations->SetOut(Location::RequiresRegister());
5865}
5866
5867void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
5868  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), GetExceptionTlsAddress());
5869}
5870
5871void LocationsBuilderX86::VisitClearException(HClearException* clear) {
5872  new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
5873}
5874
5875void InstructionCodeGeneratorX86::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
5876  __ fs()->movl(GetExceptionTlsAddress(), Immediate(0));
5877}
5878
5879void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
5880  LocationSummary* locations =
5881      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
5882  InvokeRuntimeCallingConvention calling_convention;
5883  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
5884}
5885
5886void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
5887  codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pDeliverException),
5888                          instruction,
5889                          instruction->GetDexPc(),
5890                          nullptr);
5891  CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>();
5892}
5893
5894static bool TypeCheckNeedsATemporary(TypeCheckKind type_check_kind) {
5895  return kEmitCompilerReadBarrier &&
5896      (kUseBakerReadBarrier ||
5897       type_check_kind == TypeCheckKind::kAbstractClassCheck ||
5898       type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
5899       type_check_kind == TypeCheckKind::kArrayObjectCheck);
5900}
5901
5902void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
5903  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
5904  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5905  switch (type_check_kind) {
5906    case TypeCheckKind::kExactCheck:
5907    case TypeCheckKind::kAbstractClassCheck:
5908    case TypeCheckKind::kClassHierarchyCheck:
5909    case TypeCheckKind::kArrayObjectCheck:
5910      call_kind =
5911          kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall;
5912      break;
5913    case TypeCheckKind::kArrayCheck:
5914    case TypeCheckKind::kUnresolvedCheck:
5915    case TypeCheckKind::kInterfaceCheck:
5916      call_kind = LocationSummary::kCallOnSlowPath;
5917      break;
5918  }
5919
5920  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5921  locations->SetInAt(0, Location::RequiresRegister());
5922  locations->SetInAt(1, Location::Any());
5923  // Note that TypeCheckSlowPathX86 uses this "out" register too.
5924  locations->SetOut(Location::RequiresRegister());
5925  // When read barriers are enabled, we need a temporary register for
5926  // some cases.
5927  if (TypeCheckNeedsATemporary(type_check_kind)) {
5928    locations->AddTemp(Location::RequiresRegister());
5929  }
5930}
5931
5932void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
5933  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5934  LocationSummary* locations = instruction->GetLocations();
5935  Location obj_loc = locations->InAt(0);
5936  Register obj = obj_loc.AsRegister<Register>();
5937  Location cls = locations->InAt(1);
5938  Location out_loc = locations->Out();
5939  Register out = out_loc.AsRegister<Register>();
5940  Location temp_loc = TypeCheckNeedsATemporary(type_check_kind) ?
5941      locations->GetTemp(0) :
5942      Location::NoLocation();
5943  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5944  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
5945  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
5946  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
5947  SlowPathCode* slow_path = nullptr;
5948  NearLabel done, zero;
5949
5950  // Return 0 if `obj` is null.
5951  // Avoid null check if we know obj is not null.
5952  if (instruction->MustDoNullCheck()) {
5953    __ testl(obj, obj);
5954    __ j(kEqual, &zero);
5955  }
5956
5957  // /* HeapReference<Class> */ out = obj->klass_
5958  GenerateReferenceLoadTwoRegisters(instruction, out_loc, obj_loc, class_offset, temp_loc);
5959
5960  switch (type_check_kind) {
5961    case TypeCheckKind::kExactCheck: {
5962      if (cls.IsRegister()) {
5963        __ cmpl(out, cls.AsRegister<Register>());
5964      } else {
5965        DCHECK(cls.IsStackSlot()) << cls;
5966        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5967      }
5968
5969      // Classes must be equal for the instanceof to succeed.
5970      __ j(kNotEqual, &zero);
5971      __ movl(out, Immediate(1));
5972      __ jmp(&done);
5973      break;
5974    }
5975
5976    case TypeCheckKind::kAbstractClassCheck: {
5977      // If the class is abstract, we eagerly fetch the super class of the
5978      // object to avoid doing a comparison we know will fail.
5979      NearLabel loop;
5980      __ Bind(&loop);
5981      // /* HeapReference<Class> */ out = out->super_class_
5982      GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, temp_loc);
5983      __ testl(out, out);
5984      // If `out` is null, we use it for the result, and jump to `done`.
5985      __ j(kEqual, &done);
5986      if (cls.IsRegister()) {
5987        __ cmpl(out, cls.AsRegister<Register>());
5988      } else {
5989        DCHECK(cls.IsStackSlot()) << cls;
5990        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5991      }
5992      __ j(kNotEqual, &loop);
5993      __ movl(out, Immediate(1));
5994      if (zero.IsLinked()) {
5995        __ jmp(&done);
5996      }
5997      break;
5998    }
5999
6000    case TypeCheckKind::kClassHierarchyCheck: {
6001      // Walk over the class hierarchy to find a match.
6002      NearLabel loop, success;
6003      __ Bind(&loop);
6004      if (cls.IsRegister()) {
6005        __ cmpl(out, cls.AsRegister<Register>());
6006      } else {
6007        DCHECK(cls.IsStackSlot()) << cls;
6008        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
6009      }
6010      __ j(kEqual, &success);
6011      // /* HeapReference<Class> */ out = out->super_class_
6012      GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, temp_loc);
6013      __ testl(out, out);
6014      __ j(kNotEqual, &loop);
6015      // If `out` is null, we use it for the result, and jump to `done`.
6016      __ jmp(&done);
6017      __ Bind(&success);
6018      __ movl(out, Immediate(1));
6019      if (zero.IsLinked()) {
6020        __ jmp(&done);
6021      }
6022      break;
6023    }
6024
6025    case TypeCheckKind::kArrayObjectCheck: {
6026      // Do an exact check.
6027      NearLabel exact_check;
6028      if (cls.IsRegister()) {
6029        __ cmpl(out, cls.AsRegister<Register>());
6030      } else {
6031        DCHECK(cls.IsStackSlot()) << cls;
6032        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
6033      }
6034      __ j(kEqual, &exact_check);
6035      // Otherwise, we need to check that the object's class is a non-primitive array.
6036      // /* HeapReference<Class> */ out = out->component_type_
6037      GenerateReferenceLoadOneRegister(instruction, out_loc, component_offset, temp_loc);
6038      __ testl(out, out);
6039      // If `out` is null, we use it for the result, and jump to `done`.
6040      __ j(kEqual, &done);
6041      __ cmpw(Address(out, primitive_offset), Immediate(Primitive::kPrimNot));
6042      __ j(kNotEqual, &zero);
6043      __ Bind(&exact_check);
6044      __ movl(out, Immediate(1));
6045      __ jmp(&done);
6046      break;
6047    }
6048
6049    case TypeCheckKind::kArrayCheck: {
6050      if (cls.IsRegister()) {
6051        __ cmpl(out, cls.AsRegister<Register>());
6052      } else {
6053        DCHECK(cls.IsStackSlot()) << cls;
6054        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
6055      }
6056      DCHECK(locations->OnlyCallsOnSlowPath());
6057      slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
6058                                                                    /* is_fatal */ false);
6059      codegen_->AddSlowPath(slow_path);
6060      __ j(kNotEqual, slow_path->GetEntryLabel());
6061      __ movl(out, Immediate(1));
6062      if (zero.IsLinked()) {
6063        __ jmp(&done);
6064      }
6065      break;
6066    }
6067
6068    case TypeCheckKind::kUnresolvedCheck:
6069    case TypeCheckKind::kInterfaceCheck: {
6070      // Note that we indeed only call on slow path, but we always go
6071      // into the slow path for the unresolved & interface check
6072      // cases.
6073      //
6074      // We cannot directly call the InstanceofNonTrivial runtime
6075      // entry point without resorting to a type checking slow path
6076      // here (i.e. by calling InvokeRuntime directly), as it would
6077      // require to assign fixed registers for the inputs of this
6078      // HInstanceOf instruction (following the runtime calling
6079      // convention), which might be cluttered by the potential first
6080      // read barrier emission at the beginning of this method.
6081      //
6082      // TODO: Introduce a new runtime entry point taking the object
6083      // to test (instead of its class) as argument, and let it deal
6084      // with the read barrier issues. This will let us refactor this
6085      // case of the `switch` code as it was previously (with a direct
6086      // call to the runtime not using a type checking slow path).
6087      // This should also be beneficial for the other cases above.
6088      DCHECK(locations->OnlyCallsOnSlowPath());
6089      slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
6090                                                                    /* is_fatal */ false);
6091      codegen_->AddSlowPath(slow_path);
6092      __ jmp(slow_path->GetEntryLabel());
6093      if (zero.IsLinked()) {
6094        __ jmp(&done);
6095      }
6096      break;
6097    }
6098  }
6099
6100  if (zero.IsLinked()) {
6101    __ Bind(&zero);
6102    __ xorl(out, out);
6103  }
6104
6105  if (done.IsLinked()) {
6106    __ Bind(&done);
6107  }
6108
6109  if (slow_path != nullptr) {
6110    __ Bind(slow_path->GetExitLabel());
6111  }
6112}
6113
6114void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
6115  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
6116  bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
6117  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
6118  switch (type_check_kind) {
6119    case TypeCheckKind::kExactCheck:
6120    case TypeCheckKind::kAbstractClassCheck:
6121    case TypeCheckKind::kClassHierarchyCheck:
6122    case TypeCheckKind::kArrayObjectCheck:
6123      call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ?
6124          LocationSummary::kCallOnSlowPath :
6125          LocationSummary::kNoCall;  // In fact, call on a fatal (non-returning) slow path.
6126      break;
6127    case TypeCheckKind::kArrayCheck:
6128    case TypeCheckKind::kUnresolvedCheck:
6129    case TypeCheckKind::kInterfaceCheck:
6130      call_kind = LocationSummary::kCallOnSlowPath;
6131      break;
6132  }
6133  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
6134  locations->SetInAt(0, Location::RequiresRegister());
6135  locations->SetInAt(1, Location::Any());
6136  // Note that TypeCheckSlowPathX86 uses this "temp" register too.
6137  locations->AddTemp(Location::RequiresRegister());
6138  // When read barriers are enabled, we need an additional temporary
6139  // register for some cases.
6140  if (TypeCheckNeedsATemporary(type_check_kind)) {
6141    locations->AddTemp(Location::RequiresRegister());
6142  }
6143}
6144
6145void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
6146  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
6147  LocationSummary* locations = instruction->GetLocations();
6148  Location obj_loc = locations->InAt(0);
6149  Register obj = obj_loc.AsRegister<Register>();
6150  Location cls = locations->InAt(1);
6151  Location temp_loc = locations->GetTemp(0);
6152  Register temp = temp_loc.AsRegister<Register>();
6153  Location temp2_loc = TypeCheckNeedsATemporary(type_check_kind) ?
6154      locations->GetTemp(1) :
6155      Location::NoLocation();
6156  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
6157  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
6158  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
6159  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
6160
6161  bool is_type_check_slow_path_fatal =
6162      (type_check_kind == TypeCheckKind::kExactCheck ||
6163       type_check_kind == TypeCheckKind::kAbstractClassCheck ||
6164       type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
6165       type_check_kind == TypeCheckKind::kArrayObjectCheck) &&
6166      !instruction->CanThrowIntoCatchBlock();
6167  SlowPathCode* type_check_slow_path =
6168      new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
6169                                                        is_type_check_slow_path_fatal);
6170  codegen_->AddSlowPath(type_check_slow_path);
6171
6172  NearLabel done;
6173  // Avoid null check if we know obj is not null.
6174  if (instruction->MustDoNullCheck()) {
6175    __ testl(obj, obj);
6176    __ j(kEqual, &done);
6177  }
6178
6179  // /* HeapReference<Class> */ temp = obj->klass_
6180  GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc);
6181
6182  switch (type_check_kind) {
6183    case TypeCheckKind::kExactCheck:
6184    case TypeCheckKind::kArrayCheck: {
6185      if (cls.IsRegister()) {
6186        __ cmpl(temp, cls.AsRegister<Register>());
6187      } else {
6188        DCHECK(cls.IsStackSlot()) << cls;
6189        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6190      }
6191      // Jump to slow path for throwing the exception or doing a
6192      // more involved array check.
6193      __ j(kNotEqual, type_check_slow_path->GetEntryLabel());
6194      break;
6195    }
6196
6197    case TypeCheckKind::kAbstractClassCheck: {
6198      // If the class is abstract, we eagerly fetch the super class of the
6199      // object to avoid doing a comparison we know will fail.
6200      NearLabel loop, compare_classes;
6201      __ Bind(&loop);
6202      // /* HeapReference<Class> */ temp = temp->super_class_
6203      GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, temp2_loc);
6204
6205      // If the class reference currently in `temp` is not null, jump
6206      // to the `compare_classes` label to compare it with the checked
6207      // class.
6208      __ testl(temp, temp);
6209      __ j(kNotEqual, &compare_classes);
6210      // Otherwise, jump to the slow path to throw the exception.
6211      //
6212      // But before, move back the object's class into `temp` before
6213      // going into the slow path, as it has been overwritten in the
6214      // meantime.
6215      // /* HeapReference<Class> */ temp = obj->klass_
6216      GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc);
6217      __ jmp(type_check_slow_path->GetEntryLabel());
6218
6219      __ Bind(&compare_classes);
6220      if (cls.IsRegister()) {
6221        __ cmpl(temp, cls.AsRegister<Register>());
6222      } else {
6223        DCHECK(cls.IsStackSlot()) << cls;
6224        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6225      }
6226      __ j(kNotEqual, &loop);
6227      break;
6228    }
6229
6230    case TypeCheckKind::kClassHierarchyCheck: {
6231      // Walk over the class hierarchy to find a match.
6232      NearLabel loop;
6233      __ Bind(&loop);
6234      if (cls.IsRegister()) {
6235        __ cmpl(temp, cls.AsRegister<Register>());
6236      } else {
6237        DCHECK(cls.IsStackSlot()) << cls;
6238        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6239      }
6240      __ j(kEqual, &done);
6241
6242      // /* HeapReference<Class> */ temp = temp->super_class_
6243      GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, temp2_loc);
6244
6245      // If the class reference currently in `temp` is not null, jump
6246      // back at the beginning of the loop.
6247      __ testl(temp, temp);
6248      __ j(kNotEqual, &loop);
6249      // Otherwise, jump to the slow path to throw the exception.
6250      //
6251      // But before, move back the object's class into `temp` before
6252      // going into the slow path, as it has been overwritten in the
6253      // meantime.
6254      // /* HeapReference<Class> */ temp = obj->klass_
6255      GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc);
6256      __ jmp(type_check_slow_path->GetEntryLabel());
6257      break;
6258    }
6259
6260    case TypeCheckKind::kArrayObjectCheck: {
6261      // Do an exact check.
6262      NearLabel check_non_primitive_component_type;
6263      if (cls.IsRegister()) {
6264        __ cmpl(temp, cls.AsRegister<Register>());
6265      } else {
6266        DCHECK(cls.IsStackSlot()) << cls;
6267        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6268      }
6269      __ j(kEqual, &done);
6270
6271      // Otherwise, we need to check that the object's class is a non-primitive array.
6272      // /* HeapReference<Class> */ temp = temp->component_type_
6273      GenerateReferenceLoadOneRegister(instruction, temp_loc, component_offset, temp2_loc);
6274
6275      // If the component type is not null (i.e. the object is indeed
6276      // an array), jump to label `check_non_primitive_component_type`
6277      // to further check that this component type is not a primitive
6278      // type.
6279      __ testl(temp, temp);
6280      __ j(kNotEqual, &check_non_primitive_component_type);
6281      // Otherwise, jump to the slow path to throw the exception.
6282      //
6283      // But before, move back the object's class into `temp` before
6284      // going into the slow path, as it has been overwritten in the
6285      // meantime.
6286      // /* HeapReference<Class> */ temp = obj->klass_
6287      GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc);
6288      __ jmp(type_check_slow_path->GetEntryLabel());
6289
6290      __ Bind(&check_non_primitive_component_type);
6291      __ cmpw(Address(temp, primitive_offset), Immediate(Primitive::kPrimNot));
6292      __ j(kEqual, &done);
6293      // Same comment as above regarding `temp` and the slow path.
6294      // /* HeapReference<Class> */ temp = obj->klass_
6295      GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, temp2_loc);
6296      __ jmp(type_check_slow_path->GetEntryLabel());
6297      break;
6298    }
6299
6300    case TypeCheckKind::kUnresolvedCheck:
6301    case TypeCheckKind::kInterfaceCheck:
6302      // We always go into the type check slow path for the unresolved &
6303      // interface check cases.
6304      //
6305      // We cannot directly call the CheckCast runtime entry point
6306      // without resorting to a type checking slow path here (i.e. by
6307      // calling InvokeRuntime directly), as it would require to
6308      // assign fixed registers for the inputs of this HInstanceOf
6309      // instruction (following the runtime calling convention), which
6310      // might be cluttered by the potential first read barrier
6311      // emission at the beginning of this method.
6312      //
6313      // TODO: Introduce a new runtime entry point taking the object
6314      // to test (instead of its class) as argument, and let it deal
6315      // with the read barrier issues. This will let us refactor this
6316      // case of the `switch` code as it was previously (with a direct
6317      // call to the runtime not using a type checking slow path).
6318      // This should also be beneficial for the other cases above.
6319      __ jmp(type_check_slow_path->GetEntryLabel());
6320      break;
6321  }
6322  __ Bind(&done);
6323
6324  __ Bind(type_check_slow_path->GetExitLabel());
6325}
6326
6327void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
6328  LocationSummary* locations =
6329      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
6330  InvokeRuntimeCallingConvention calling_convention;
6331  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
6332}
6333
6334void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
6335  codegen_->InvokeRuntime(instruction->IsEnter() ? QUICK_ENTRY_POINT(pLockObject)
6336                                                 : QUICK_ENTRY_POINT(pUnlockObject),
6337                          instruction,
6338                          instruction->GetDexPc(),
6339                          nullptr);
6340  if (instruction->IsEnter()) {
6341    CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>();
6342  } else {
6343    CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>();
6344  }
6345}
6346
6347void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
6348void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
6349void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
6350
6351void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
6352  LocationSummary* locations =
6353      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
6354  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
6355         || instruction->GetResultType() == Primitive::kPrimLong);
6356  locations->SetInAt(0, Location::RequiresRegister());
6357  locations->SetInAt(1, Location::Any());
6358  locations->SetOut(Location::SameAsFirstInput());
6359}
6360
6361void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
6362  HandleBitwiseOperation(instruction);
6363}
6364
6365void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
6366  HandleBitwiseOperation(instruction);
6367}
6368
6369void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
6370  HandleBitwiseOperation(instruction);
6371}
6372
6373void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
6374  LocationSummary* locations = instruction->GetLocations();
6375  Location first = locations->InAt(0);
6376  Location second = locations->InAt(1);
6377  DCHECK(first.Equals(locations->Out()));
6378
6379  if (instruction->GetResultType() == Primitive::kPrimInt) {
6380    if (second.IsRegister()) {
6381      if (instruction->IsAnd()) {
6382        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
6383      } else if (instruction->IsOr()) {
6384        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
6385      } else {
6386        DCHECK(instruction->IsXor());
6387        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
6388      }
6389    } else if (second.IsConstant()) {
6390      if (instruction->IsAnd()) {
6391        __ andl(first.AsRegister<Register>(),
6392                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6393      } else if (instruction->IsOr()) {
6394        __ orl(first.AsRegister<Register>(),
6395               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6396      } else {
6397        DCHECK(instruction->IsXor());
6398        __ xorl(first.AsRegister<Register>(),
6399                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6400      }
6401    } else {
6402      if (instruction->IsAnd()) {
6403        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6404      } else if (instruction->IsOr()) {
6405        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6406      } else {
6407        DCHECK(instruction->IsXor());
6408        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6409      }
6410    }
6411  } else {
6412    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
6413    if (second.IsRegisterPair()) {
6414      if (instruction->IsAnd()) {
6415        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6416        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6417      } else if (instruction->IsOr()) {
6418        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6419        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6420      } else {
6421        DCHECK(instruction->IsXor());
6422        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6423        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6424      }
6425    } else if (second.IsDoubleStackSlot()) {
6426      if (instruction->IsAnd()) {
6427        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6428        __ andl(first.AsRegisterPairHigh<Register>(),
6429                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6430      } else if (instruction->IsOr()) {
6431        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6432        __ orl(first.AsRegisterPairHigh<Register>(),
6433                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6434      } else {
6435        DCHECK(instruction->IsXor());
6436        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6437        __ xorl(first.AsRegisterPairHigh<Register>(),
6438                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6439      }
6440    } else {
6441      DCHECK(second.IsConstant()) << second;
6442      int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
6443      int32_t low_value = Low32Bits(value);
6444      int32_t high_value = High32Bits(value);
6445      Immediate low(low_value);
6446      Immediate high(high_value);
6447      Register first_low = first.AsRegisterPairLow<Register>();
6448      Register first_high = first.AsRegisterPairHigh<Register>();
6449      if (instruction->IsAnd()) {
6450        if (low_value == 0) {
6451          __ xorl(first_low, first_low);
6452        } else if (low_value != -1) {
6453          __ andl(first_low, low);
6454        }
6455        if (high_value == 0) {
6456          __ xorl(first_high, first_high);
6457        } else if (high_value != -1) {
6458          __ andl(first_high, high);
6459        }
6460      } else if (instruction->IsOr()) {
6461        if (low_value != 0) {
6462          __ orl(first_low, low);
6463        }
6464        if (high_value != 0) {
6465          __ orl(first_high, high);
6466        }
6467      } else {
6468        DCHECK(instruction->IsXor());
6469        if (low_value != 0) {
6470          __ xorl(first_low, low);
6471        }
6472        if (high_value != 0) {
6473          __ xorl(first_high, high);
6474        }
6475      }
6476    }
6477  }
6478}
6479
6480void InstructionCodeGeneratorX86::GenerateReferenceLoadOneRegister(HInstruction* instruction,
6481                                                                   Location out,
6482                                                                   uint32_t offset,
6483                                                                   Location temp) {
6484  Register out_reg = out.AsRegister<Register>();
6485  if (kEmitCompilerReadBarrier) {
6486    if (kUseBakerReadBarrier) {
6487      // Load with fast path based Baker's read barrier.
6488      // /* HeapReference<Object> */ out = *(out + offset)
6489      codegen_->GenerateFieldLoadWithBakerReadBarrier(
6490          instruction, out, out_reg, offset, temp, /* needs_null_check */ false);
6491    } else {
6492      // Load with slow path based read barrier.
6493      // Save the value of `out` into `temp` before overwriting it
6494      // in the following move operation, as we will need it for the
6495      // read barrier below.
6496      __ movl(temp.AsRegister<Register>(), out_reg);
6497      // /* HeapReference<Object> */ out = *(out + offset)
6498      __ movl(out_reg, Address(out_reg, offset));
6499      codegen_->GenerateReadBarrierSlow(instruction, out, out, temp, offset);
6500    }
6501  } else {
6502    // Plain load with no read barrier.
6503    // /* HeapReference<Object> */ out = *(out + offset)
6504    __ movl(out_reg, Address(out_reg, offset));
6505    __ MaybeUnpoisonHeapReference(out_reg);
6506  }
6507}
6508
6509void InstructionCodeGeneratorX86::GenerateReferenceLoadTwoRegisters(HInstruction* instruction,
6510                                                                    Location out,
6511                                                                    Location obj,
6512                                                                    uint32_t offset,
6513                                                                    Location temp) {
6514  Register out_reg = out.AsRegister<Register>();
6515  Register obj_reg = obj.AsRegister<Register>();
6516  if (kEmitCompilerReadBarrier) {
6517    if (kUseBakerReadBarrier) {
6518      // Load with fast path based Baker's read barrier.
6519      // /* HeapReference<Object> */ out = *(obj + offset)
6520      codegen_->GenerateFieldLoadWithBakerReadBarrier(
6521          instruction, out, obj_reg, offset, temp, /* needs_null_check */ false);
6522    } else {
6523      // Load with slow path based read barrier.
6524      // /* HeapReference<Object> */ out = *(obj + offset)
6525      __ movl(out_reg, Address(obj_reg, offset));
6526      codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset);
6527    }
6528  } else {
6529    // Plain load with no read barrier.
6530    // /* HeapReference<Object> */ out = *(obj + offset)
6531    __ movl(out_reg, Address(obj_reg, offset));
6532    __ MaybeUnpoisonHeapReference(out_reg);
6533  }
6534}
6535
6536void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad(HInstruction* instruction,
6537                                                          Location root,
6538                                                          Register obj,
6539                                                          uint32_t offset) {
6540  Register root_reg = root.AsRegister<Register>();
6541  if (kEmitCompilerReadBarrier) {
6542    if (kUseBakerReadBarrier) {
6543      // Fast path implementation of art::ReadBarrier::BarrierForRoot when
6544      // Baker's read barrier are used:
6545      //
6546      //   root = obj.field;
6547      //   if (Thread::Current()->GetIsGcMarking()) {
6548      //     root = ReadBarrier::Mark(root)
6549      //   }
6550
6551      // /* GcRoot<mirror::Object> */ root = *(obj + offset)
6552      __ movl(root_reg, Address(obj, offset));
6553      static_assert(
6554          sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>),
6555          "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> "
6556          "have different sizes.");
6557      static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t),
6558                    "art::mirror::CompressedReference<mirror::Object> and int32_t "
6559                    "have different sizes.");
6560
6561      // Slow path used to mark the GC root `root`.
6562      SlowPathCode* slow_path =
6563          new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86(instruction, root, root);
6564      codegen_->AddSlowPath(slow_path);
6565
6566      __ fs()->cmpl(Address::Absolute(Thread::IsGcMarkingOffset<kX86WordSize>().Int32Value()),
6567                    Immediate(0));
6568      __ j(kNotEqual, slow_path->GetEntryLabel());
6569      __ Bind(slow_path->GetExitLabel());
6570    } else {
6571      // GC root loaded through a slow path for read barriers other
6572      // than Baker's.
6573      // /* GcRoot<mirror::Object>* */ root = obj + offset
6574      __ leal(root_reg, Address(obj, offset));
6575      // /* mirror::Object* */ root = root->Read()
6576      codegen_->GenerateReadBarrierForRootSlow(instruction, root, root);
6577    }
6578  } else {
6579    // Plain GC root load with no read barrier.
6580    // /* GcRoot<mirror::Object> */ root = *(obj + offset)
6581    __ movl(root_reg, Address(obj, offset));
6582  }
6583}
6584
6585void CodeGeneratorX86::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
6586                                                             Location ref,
6587                                                             Register obj,
6588                                                             uint32_t offset,
6589                                                             Location temp,
6590                                                             bool needs_null_check) {
6591  DCHECK(kEmitCompilerReadBarrier);
6592  DCHECK(kUseBakerReadBarrier);
6593
6594  // /* HeapReference<Object> */ ref = *(obj + offset)
6595  Address src(obj, offset);
6596  GenerateReferenceLoadWithBakerReadBarrier(instruction, ref, obj, src, temp, needs_null_check);
6597}
6598
6599void CodeGeneratorX86::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction,
6600                                                             Location ref,
6601                                                             Register obj,
6602                                                             uint32_t data_offset,
6603                                                             Location index,
6604                                                             Location temp,
6605                                                             bool needs_null_check) {
6606  DCHECK(kEmitCompilerReadBarrier);
6607  DCHECK(kUseBakerReadBarrier);
6608
6609  // /* HeapReference<Object> */ ref =
6610  //     *(obj + data_offset + index * sizeof(HeapReference<Object>))
6611  Address src = index.IsConstant() ?
6612      Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset) :
6613      Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset);
6614  GenerateReferenceLoadWithBakerReadBarrier(instruction, ref, obj, src, temp, needs_null_check);
6615}
6616
6617void CodeGeneratorX86::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction,
6618                                                                 Location ref,
6619                                                                 Register obj,
6620                                                                 const Address& src,
6621                                                                 Location temp,
6622                                                                 bool needs_null_check) {
6623  DCHECK(kEmitCompilerReadBarrier);
6624  DCHECK(kUseBakerReadBarrier);
6625
6626  // In slow path based read barriers, the read barrier call is
6627  // inserted after the original load. However, in fast path based
6628  // Baker's read barriers, we need to perform the load of
6629  // mirror::Object::monitor_ *before* the original reference load.
6630  // This load-load ordering is required by the read barrier.
6631  // The fast path/slow path (for Baker's algorithm) should look like:
6632  //
6633  //   uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
6634  //   lfence;  // Load fence or artificial data dependency to prevent load-load reordering
6635  //   HeapReference<Object> ref = *src;  // Original reference load.
6636  //   bool is_gray = (rb_state == ReadBarrier::gray_ptr_);
6637  //   if (is_gray) {
6638  //     ref = ReadBarrier::Mark(ref);  // Performed by runtime entrypoint slow path.
6639  //   }
6640  //
6641  // Note: the original implementation in ReadBarrier::Barrier is
6642  // slightly more complex as:
6643  // - it implements the load-load fence using a data dependency on
6644  //   the high-bits of rb_state, which are expected to be all zeroes;
6645  // - it performs additional checks that we do not do here for
6646  //   performance reasons.
6647
6648  Register ref_reg = ref.AsRegister<Register>();
6649  Register temp_reg = temp.AsRegister<Register>();
6650  uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
6651
6652  // /* int32_t */ monitor = obj->monitor_
6653  __ movl(temp_reg, Address(obj, monitor_offset));
6654  if (needs_null_check) {
6655    MaybeRecordImplicitNullCheck(instruction);
6656  }
6657  // /* LockWord */ lock_word = LockWord(monitor)
6658  static_assert(sizeof(LockWord) == sizeof(int32_t),
6659                "art::LockWord and int32_t have different sizes.");
6660  // /* uint32_t */ rb_state = lock_word.ReadBarrierState()
6661  __ shrl(temp_reg, Immediate(LockWord::kReadBarrierStateShift));
6662  __ andl(temp_reg, Immediate(LockWord::kReadBarrierStateMask));
6663  static_assert(
6664      LockWord::kReadBarrierStateMask == ReadBarrier::rb_ptr_mask_,
6665      "art::LockWord::kReadBarrierStateMask is not equal to art::ReadBarrier::rb_ptr_mask_.");
6666
6667  // Load fence to prevent load-load reordering.
6668  // Note that this is a no-op, thanks to the x86 memory model.
6669  GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
6670
6671  // The actual reference load.
6672  // /* HeapReference<Object> */ ref = *src
6673  __ movl(ref_reg, src);
6674
6675  // Object* ref = ref_addr->AsMirrorPtr()
6676  __ MaybeUnpoisonHeapReference(ref_reg);
6677
6678  // Slow path used to mark the object `ref` when it is gray.
6679  SlowPathCode* slow_path =
6680      new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86(instruction, ref, ref);
6681  AddSlowPath(slow_path);
6682
6683  // if (rb_state == ReadBarrier::gray_ptr_)
6684  //   ref = ReadBarrier::Mark(ref);
6685  __ cmpl(temp_reg, Immediate(ReadBarrier::gray_ptr_));
6686  __ j(kEqual, slow_path->GetEntryLabel());
6687  __ Bind(slow_path->GetExitLabel());
6688}
6689
6690void CodeGeneratorX86::GenerateReadBarrierSlow(HInstruction* instruction,
6691                                               Location out,
6692                                               Location ref,
6693                                               Location obj,
6694                                               uint32_t offset,
6695                                               Location index) {
6696  DCHECK(kEmitCompilerReadBarrier);
6697
6698  // Insert a slow path based read barrier *after* the reference load.
6699  //
6700  // If heap poisoning is enabled, the unpoisoning of the loaded
6701  // reference will be carried out by the runtime within the slow
6702  // path.
6703  //
6704  // Note that `ref` currently does not get unpoisoned (when heap
6705  // poisoning is enabled), which is alright as the `ref` argument is
6706  // not used by the artReadBarrierSlow entry point.
6707  //
6708  // TODO: Unpoison `ref` when it is used by artReadBarrierSlow.
6709  SlowPathCode* slow_path = new (GetGraph()->GetArena())
6710      ReadBarrierForHeapReferenceSlowPathX86(instruction, out, ref, obj, offset, index);
6711  AddSlowPath(slow_path);
6712
6713  __ jmp(slow_path->GetEntryLabel());
6714  __ Bind(slow_path->GetExitLabel());
6715}
6716
6717void CodeGeneratorX86::MaybeGenerateReadBarrierSlow(HInstruction* instruction,
6718                                                    Location out,
6719                                                    Location ref,
6720                                                    Location obj,
6721                                                    uint32_t offset,
6722                                                    Location index) {
6723  if (kEmitCompilerReadBarrier) {
6724    // Baker's read barriers shall be handled by the fast path
6725    // (CodeGeneratorX86::GenerateReferenceLoadWithBakerReadBarrier).
6726    DCHECK(!kUseBakerReadBarrier);
6727    // If heap poisoning is enabled, unpoisoning will be taken care of
6728    // by the runtime within the slow path.
6729    GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index);
6730  } else if (kPoisonHeapReferences) {
6731    __ UnpoisonHeapReference(out.AsRegister<Register>());
6732  }
6733}
6734
6735void CodeGeneratorX86::GenerateReadBarrierForRootSlow(HInstruction* instruction,
6736                                                      Location out,
6737                                                      Location root) {
6738  DCHECK(kEmitCompilerReadBarrier);
6739
6740  // Insert a slow path based read barrier *after* the GC root load.
6741  //
6742  // Note that GC roots are not affected by heap poisoning, so we do
6743  // not need to do anything special for this here.
6744  SlowPathCode* slow_path =
6745      new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathX86(instruction, out, root);
6746  AddSlowPath(slow_path);
6747
6748  __ jmp(slow_path->GetEntryLabel());
6749  __ Bind(slow_path->GetExitLabel());
6750}
6751
6752void LocationsBuilderX86::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
6753  // Nothing to do, this should be removed during prepare for register allocator.
6754  LOG(FATAL) << "Unreachable";
6755}
6756
6757void InstructionCodeGeneratorX86::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
6758  // Nothing to do, this should be removed during prepare for register allocator.
6759  LOG(FATAL) << "Unreachable";
6760}
6761
6762// Simple implementation of packed switch - generate cascaded compare/jumps.
6763void LocationsBuilderX86::VisitPackedSwitch(HPackedSwitch* switch_instr) {
6764  LocationSummary* locations =
6765      new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
6766  locations->SetInAt(0, Location::RequiresRegister());
6767}
6768
6769void InstructionCodeGeneratorX86::GenPackedSwitchWithCompares(Register value_reg,
6770                                                              int32_t lower_bound,
6771                                                              uint32_t num_entries,
6772                                                              HBasicBlock* switch_block,
6773                                                              HBasicBlock* default_block) {
6774  // Figure out the correct compare values and jump conditions.
6775  // Handle the first compare/branch as a special case because it might
6776  // jump to the default case.
6777  DCHECK_GT(num_entries, 2u);
6778  Condition first_condition;
6779  uint32_t index;
6780  const ArenaVector<HBasicBlock*>& successors = switch_block->GetSuccessors();
6781  if (lower_bound != 0) {
6782    first_condition = kLess;
6783    __ cmpl(value_reg, Immediate(lower_bound));
6784    __ j(first_condition, codegen_->GetLabelOf(default_block));
6785    __ j(kEqual, codegen_->GetLabelOf(successors[0]));
6786
6787    index = 1;
6788  } else {
6789    // Handle all the compare/jumps below.
6790    first_condition = kBelow;
6791    index = 0;
6792  }
6793
6794  // Handle the rest of the compare/jumps.
6795  for (; index + 1 < num_entries; index += 2) {
6796    int32_t compare_to_value = lower_bound + index + 1;
6797    __ cmpl(value_reg, Immediate(compare_to_value));
6798    // Jump to successors[index] if value < case_value[index].
6799    __ j(first_condition, codegen_->GetLabelOf(successors[index]));
6800    // Jump to successors[index + 1] if value == case_value[index + 1].
6801    __ j(kEqual, codegen_->GetLabelOf(successors[index + 1]));
6802  }
6803
6804  if (index != num_entries) {
6805    // There are an odd number of entries. Handle the last one.
6806    DCHECK_EQ(index + 1, num_entries);
6807    __ cmpl(value_reg, Immediate(lower_bound + index));
6808    __ j(kEqual, codegen_->GetLabelOf(successors[index]));
6809  }
6810
6811  // And the default for any other value.
6812  if (!codegen_->GoesToNextBlock(switch_block, default_block)) {
6813    __ jmp(codegen_->GetLabelOf(default_block));
6814  }
6815}
6816
6817void InstructionCodeGeneratorX86::VisitPackedSwitch(HPackedSwitch* switch_instr) {
6818  int32_t lower_bound = switch_instr->GetStartValue();
6819  uint32_t num_entries = switch_instr->GetNumEntries();
6820  LocationSummary* locations = switch_instr->GetLocations();
6821  Register value_reg = locations->InAt(0).AsRegister<Register>();
6822
6823  GenPackedSwitchWithCompares(value_reg,
6824                              lower_bound,
6825                              num_entries,
6826                              switch_instr->GetBlock(),
6827                              switch_instr->GetDefaultBlock());
6828}
6829
6830void LocationsBuilderX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) {
6831  LocationSummary* locations =
6832      new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
6833  locations->SetInAt(0, Location::RequiresRegister());
6834
6835  // Constant area pointer.
6836  locations->SetInAt(1, Location::RequiresRegister());
6837
6838  // And the temporary we need.
6839  locations->AddTemp(Location::RequiresRegister());
6840}
6841
6842void InstructionCodeGeneratorX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) {
6843  int32_t lower_bound = switch_instr->GetStartValue();
6844  uint32_t num_entries = switch_instr->GetNumEntries();
6845  LocationSummary* locations = switch_instr->GetLocations();
6846  Register value_reg = locations->InAt(0).AsRegister<Register>();
6847  HBasicBlock* default_block = switch_instr->GetDefaultBlock();
6848
6849  if (num_entries <= kPackedSwitchJumpTableThreshold) {
6850    GenPackedSwitchWithCompares(value_reg,
6851                                lower_bound,
6852                                num_entries,
6853                                switch_instr->GetBlock(),
6854                                default_block);
6855    return;
6856  }
6857
6858  // Optimizing has a jump area.
6859  Register temp_reg = locations->GetTemp(0).AsRegister<Register>();
6860  Register constant_area = locations->InAt(1).AsRegister<Register>();
6861
6862  // Remove the bias, if needed.
6863  if (lower_bound != 0) {
6864    __ leal(temp_reg, Address(value_reg, -lower_bound));
6865    value_reg = temp_reg;
6866  }
6867
6868  // Is the value in range?
6869  DCHECK_GE(num_entries, 1u);
6870  __ cmpl(value_reg, Immediate(num_entries - 1));
6871  __ j(kAbove, codegen_->GetLabelOf(default_block));
6872
6873  // We are in the range of the table.
6874  // Load (target-constant_area) from the jump table, indexing by the value.
6875  __ movl(temp_reg, codegen_->LiteralCaseTable(switch_instr, constant_area, value_reg));
6876
6877  // Compute the actual target address by adding in constant_area.
6878  __ addl(temp_reg, constant_area);
6879
6880  // And jump.
6881  __ jmp(temp_reg);
6882}
6883
6884void LocationsBuilderX86::VisitX86ComputeBaseMethodAddress(
6885    HX86ComputeBaseMethodAddress* insn) {
6886  LocationSummary* locations =
6887      new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
6888  locations->SetOut(Location::RequiresRegister());
6889}
6890
6891void InstructionCodeGeneratorX86::VisitX86ComputeBaseMethodAddress(
6892    HX86ComputeBaseMethodAddress* insn) {
6893  LocationSummary* locations = insn->GetLocations();
6894  Register reg = locations->Out().AsRegister<Register>();
6895
6896  // Generate call to next instruction.
6897  Label next_instruction;
6898  __ call(&next_instruction);
6899  __ Bind(&next_instruction);
6900
6901  // Remember this offset for later use with constant area.
6902  codegen_->SetMethodAddressOffset(GetAssembler()->CodeSize());
6903
6904  // Grab the return address off the stack.
6905  __ popl(reg);
6906}
6907
6908void LocationsBuilderX86::VisitX86LoadFromConstantTable(
6909    HX86LoadFromConstantTable* insn) {
6910  LocationSummary* locations =
6911      new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
6912
6913  locations->SetInAt(0, Location::RequiresRegister());
6914  locations->SetInAt(1, Location::ConstantLocation(insn->GetConstant()));
6915
6916  // If we don't need to be materialized, we only need the inputs to be set.
6917  if (!insn->NeedsMaterialization()) {
6918    return;
6919  }
6920
6921  switch (insn->GetType()) {
6922    case Primitive::kPrimFloat:
6923    case Primitive::kPrimDouble:
6924      locations->SetOut(Location::RequiresFpuRegister());
6925      break;
6926
6927    case Primitive::kPrimInt:
6928      locations->SetOut(Location::RequiresRegister());
6929      break;
6930
6931    default:
6932      LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
6933  }
6934}
6935
6936void InstructionCodeGeneratorX86::VisitX86LoadFromConstantTable(HX86LoadFromConstantTable* insn) {
6937  if (!insn->NeedsMaterialization()) {
6938    return;
6939  }
6940
6941  LocationSummary* locations = insn->GetLocations();
6942  Location out = locations->Out();
6943  Register const_area = locations->InAt(0).AsRegister<Register>();
6944  HConstant *value = insn->GetConstant();
6945
6946  switch (insn->GetType()) {
6947    case Primitive::kPrimFloat:
6948      __ movss(out.AsFpuRegister<XmmRegister>(),
6949               codegen_->LiteralFloatAddress(value->AsFloatConstant()->GetValue(), const_area));
6950      break;
6951
6952    case Primitive::kPrimDouble:
6953      __ movsd(out.AsFpuRegister<XmmRegister>(),
6954               codegen_->LiteralDoubleAddress(value->AsDoubleConstant()->GetValue(), const_area));
6955      break;
6956
6957    case Primitive::kPrimInt:
6958      __ movl(out.AsRegister<Register>(),
6959              codegen_->LiteralInt32Address(value->AsIntConstant()->GetValue(), const_area));
6960      break;
6961
6962    default:
6963      LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
6964  }
6965}
6966
6967/**
6968 * Class to handle late fixup of offsets into constant area.
6969 */
6970class RIPFixup : public AssemblerFixup, public ArenaObject<kArenaAllocCodeGenerator> {
6971 public:
6972  RIPFixup(CodeGeneratorX86& codegen, size_t offset)
6973      : codegen_(&codegen), offset_into_constant_area_(offset) {}
6974
6975 protected:
6976  void SetOffset(size_t offset) { offset_into_constant_area_ = offset; }
6977
6978  CodeGeneratorX86* codegen_;
6979
6980 private:
6981  void Process(const MemoryRegion& region, int pos) OVERRIDE {
6982    // Patch the correct offset for the instruction.  The place to patch is the
6983    // last 4 bytes of the instruction.
6984    // The value to patch is the distance from the offset in the constant area
6985    // from the address computed by the HX86ComputeBaseMethodAddress instruction.
6986    int32_t constant_offset = codegen_->ConstantAreaStart() + offset_into_constant_area_;
6987    int32_t relative_position = constant_offset - codegen_->GetMethodAddressOffset();;
6988
6989    // Patch in the right value.
6990    region.StoreUnaligned<int32_t>(pos - 4, relative_position);
6991  }
6992
6993  // Location in constant area that the fixup refers to.
6994  int32_t offset_into_constant_area_;
6995};
6996
6997/**
6998 * Class to handle late fixup of offsets to a jump table that will be created in the
6999 * constant area.
7000 */
7001class JumpTableRIPFixup : public RIPFixup {
7002 public:
7003  JumpTableRIPFixup(CodeGeneratorX86& codegen, HX86PackedSwitch* switch_instr)
7004      : RIPFixup(codegen, static_cast<size_t>(-1)), switch_instr_(switch_instr) {}
7005
7006  void CreateJumpTable() {
7007    X86Assembler* assembler = codegen_->GetAssembler();
7008
7009    // Ensure that the reference to the jump table has the correct offset.
7010    const int32_t offset_in_constant_table = assembler->ConstantAreaSize();
7011    SetOffset(offset_in_constant_table);
7012
7013    // The label values in the jump table are computed relative to the
7014    // instruction addressing the constant area.
7015    const int32_t relative_offset = codegen_->GetMethodAddressOffset();
7016
7017    // Populate the jump table with the correct values for the jump table.
7018    int32_t num_entries = switch_instr_->GetNumEntries();
7019    HBasicBlock* block = switch_instr_->GetBlock();
7020    const ArenaVector<HBasicBlock*>& successors = block->GetSuccessors();
7021    // The value that we want is the target offset - the position of the table.
7022    for (int32_t i = 0; i < num_entries; i++) {
7023      HBasicBlock* b = successors[i];
7024      Label* l = codegen_->GetLabelOf(b);
7025      DCHECK(l->IsBound());
7026      int32_t offset_to_block = l->Position() - relative_offset;
7027      assembler->AppendInt32(offset_to_block);
7028    }
7029  }
7030
7031 private:
7032  const HX86PackedSwitch* switch_instr_;
7033};
7034
7035void CodeGeneratorX86::Finalize(CodeAllocator* allocator) {
7036  // Generate the constant area if needed.
7037  X86Assembler* assembler = GetAssembler();
7038  if (!assembler->IsConstantAreaEmpty() || !fixups_to_jump_tables_.empty()) {
7039    // Align to 4 byte boundary to reduce cache misses, as the data is 4 and 8
7040    // byte values.
7041    assembler->Align(4, 0);
7042    constant_area_start_ = assembler->CodeSize();
7043
7044    // Populate any jump tables.
7045    for (auto jump_table : fixups_to_jump_tables_) {
7046      jump_table->CreateJumpTable();
7047    }
7048
7049    // And now add the constant area to the generated code.
7050    assembler->AddConstantArea();
7051  }
7052
7053  // And finish up.
7054  CodeGenerator::Finalize(allocator);
7055}
7056
7057Address CodeGeneratorX86::LiteralDoubleAddress(double v, Register reg) {
7058  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddDouble(v));
7059  return Address(reg, kDummy32BitOffset, fixup);
7060}
7061
7062Address CodeGeneratorX86::LiteralFloatAddress(float v, Register reg) {
7063  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddFloat(v));
7064  return Address(reg, kDummy32BitOffset, fixup);
7065}
7066
7067Address CodeGeneratorX86::LiteralInt32Address(int32_t v, Register reg) {
7068  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt32(v));
7069  return Address(reg, kDummy32BitOffset, fixup);
7070}
7071
7072Address CodeGeneratorX86::LiteralInt64Address(int64_t v, Register reg) {
7073  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt64(v));
7074  return Address(reg, kDummy32BitOffset, fixup);
7075}
7076
7077Address CodeGeneratorX86::LiteralCaseTable(HX86PackedSwitch* switch_instr,
7078                                           Register reg,
7079                                           Register value) {
7080  // Create a fixup to be used to create and address the jump table.
7081  JumpTableRIPFixup* table_fixup =
7082      new (GetGraph()->GetArena()) JumpTableRIPFixup(*this, switch_instr);
7083
7084  // We have to populate the jump tables.
7085  fixups_to_jump_tables_.push_back(table_fixup);
7086
7087  // We want a scaled address, as we are extracting the correct offset from the table.
7088  return Address(reg, value, TIMES_4, kDummy32BitOffset, table_fixup);
7089}
7090
7091// TODO: target as memory.
7092void CodeGeneratorX86::MoveFromReturnRegister(Location target, Primitive::Type type) {
7093  if (!target.IsValid()) {
7094    DCHECK_EQ(type, Primitive::kPrimVoid);
7095    return;
7096  }
7097
7098  DCHECK_NE(type, Primitive::kPrimVoid);
7099
7100  Location return_loc = InvokeDexCallingConventionVisitorX86().GetReturnLocation(type);
7101  if (target.Equals(return_loc)) {
7102    return;
7103  }
7104
7105  // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged
7106  //       with the else branch.
7107  if (type == Primitive::kPrimLong) {
7108    HParallelMove parallel_move(GetGraph()->GetArena());
7109    parallel_move.AddMove(return_loc.ToLow(), target.ToLow(), Primitive::kPrimInt, nullptr);
7110    parallel_move.AddMove(return_loc.ToHigh(), target.ToHigh(), Primitive::kPrimInt, nullptr);
7111    GetMoveResolver()->EmitNativeCode(&parallel_move);
7112  } else {
7113    // Let the parallel move resolver take care of all of this.
7114    HParallelMove parallel_move(GetGraph()->GetArena());
7115    parallel_move.AddMove(return_loc, target, type, nullptr);
7116    GetMoveResolver()->EmitNativeCode(&parallel_move);
7117  }
7118}
7119
7120#undef __
7121
7122}  // namespace x86
7123}  // namespace art
7124