code_generator_x86.cc revision c88ef3a10c474045a3476a02ae75d07ddd3230b7 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "art_method.h"
20#include "code_generator_utils.h"
21#include "compiled_method.h"
22#include "entrypoints/quick/quick_entrypoints.h"
23#include "entrypoints/quick/quick_entrypoints_enum.h"
24#include "gc/accounting/card_table.h"
25#include "intrinsics.h"
26#include "intrinsics_x86.h"
27#include "mirror/array-inl.h"
28#include "mirror/class-inl.h"
29#include "pc_relative_fixups_x86.h"
30#include "thread.h"
31#include "utils/assembler.h"
32#include "utils/stack_checks.h"
33#include "utils/x86/assembler_x86.h"
34#include "utils/x86/managed_register_x86.h"
35
36namespace art {
37
38template<class MirrorType>
39class GcRoot;
40
41namespace x86 {
42
43static constexpr int kCurrentMethodStackOffset = 0;
44static constexpr Register kMethodRegisterArgument = EAX;
45
46static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI };
47
48static constexpr int kC2ConditionMask = 0x400;
49
50static constexpr int kFakeReturnRegister = Register(8);
51
52#define __ down_cast<X86Assembler*>(codegen->GetAssembler())->
53#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kX86WordSize, x).Int32Value()
54
55class NullCheckSlowPathX86 : public SlowPathCode {
56 public:
57  explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {}
58
59  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
60    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
61    __ Bind(GetEntryLabel());
62    if (instruction_->CanThrowIntoCatchBlock()) {
63      // Live registers will be restored in the catch block if caught.
64      SaveLiveRegisters(codegen, instruction_->GetLocations());
65    }
66    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowNullPointer),
67                               instruction_,
68                               instruction_->GetDexPc(),
69                               this);
70    CheckEntrypointTypes<kQuickThrowNullPointer, void, void>();
71  }
72
73  bool IsFatal() const OVERRIDE { return true; }
74
75  const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathX86"; }
76
77 private:
78  HNullCheck* const instruction_;
79  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
80};
81
82class DivZeroCheckSlowPathX86 : public SlowPathCode {
83 public:
84  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {}
85
86  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
87    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
88    __ Bind(GetEntryLabel());
89    if (instruction_->CanThrowIntoCatchBlock()) {
90      // Live registers will be restored in the catch block if caught.
91      SaveLiveRegisters(codegen, instruction_->GetLocations());
92    }
93    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowDivZero),
94                               instruction_,
95                               instruction_->GetDexPc(),
96                               this);
97    CheckEntrypointTypes<kQuickThrowDivZero, void, void>();
98  }
99
100  bool IsFatal() const OVERRIDE { return true; }
101
102  const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathX86"; }
103
104 private:
105  HDivZeroCheck* const instruction_;
106  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
107};
108
109class DivRemMinusOneSlowPathX86 : public SlowPathCode {
110 public:
111  DivRemMinusOneSlowPathX86(Register reg, bool is_div) : reg_(reg), is_div_(is_div) {}
112
113  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
114    __ Bind(GetEntryLabel());
115    if (is_div_) {
116      __ negl(reg_);
117    } else {
118      __ movl(reg_, Immediate(0));
119    }
120    __ jmp(GetExitLabel());
121  }
122
123  const char* GetDescription() const OVERRIDE { return "DivRemMinusOneSlowPathX86"; }
124
125 private:
126  Register reg_;
127  bool is_div_;
128  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
129};
130
131class BoundsCheckSlowPathX86 : public SlowPathCode {
132 public:
133  explicit BoundsCheckSlowPathX86(HBoundsCheck* instruction) : instruction_(instruction) {}
134
135  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
136    LocationSummary* locations = instruction_->GetLocations();
137    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
138    __ Bind(GetEntryLabel());
139    // We're moving two locations to locations that could overlap, so we need a parallel
140    // move resolver.
141    if (instruction_->CanThrowIntoCatchBlock()) {
142      // Live registers will be restored in the catch block if caught.
143      SaveLiveRegisters(codegen, instruction_->GetLocations());
144    }
145    InvokeRuntimeCallingConvention calling_convention;
146    x86_codegen->EmitParallelMoves(
147        locations->InAt(0),
148        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
149        Primitive::kPrimInt,
150        locations->InAt(1),
151        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
152        Primitive::kPrimInt);
153    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowArrayBounds),
154                               instruction_,
155                               instruction_->GetDexPc(),
156                               this);
157    CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>();
158  }
159
160  bool IsFatal() const OVERRIDE { return true; }
161
162  const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathX86"; }
163
164 private:
165  HBoundsCheck* const instruction_;
166
167  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
168};
169
170class SuspendCheckSlowPathX86 : public SlowPathCode {
171 public:
172  SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
173      : instruction_(instruction), successor_(successor) {}
174
175  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
176    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
177    __ Bind(GetEntryLabel());
178    SaveLiveRegisters(codegen, instruction_->GetLocations());
179    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pTestSuspend),
180                               instruction_,
181                               instruction_->GetDexPc(),
182                               this);
183    CheckEntrypointTypes<kQuickTestSuspend, void, void>();
184    RestoreLiveRegisters(codegen, instruction_->GetLocations());
185    if (successor_ == nullptr) {
186      __ jmp(GetReturnLabel());
187    } else {
188      __ jmp(x86_codegen->GetLabelOf(successor_));
189    }
190  }
191
192  Label* GetReturnLabel() {
193    DCHECK(successor_ == nullptr);
194    return &return_label_;
195  }
196
197  HBasicBlock* GetSuccessor() const {
198    return successor_;
199  }
200
201  const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathX86"; }
202
203 private:
204  HSuspendCheck* const instruction_;
205  HBasicBlock* const successor_;
206  Label return_label_;
207
208  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
209};
210
211class LoadStringSlowPathX86 : public SlowPathCode {
212 public:
213  explicit LoadStringSlowPathX86(HLoadString* instruction) : instruction_(instruction) {}
214
215  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
216    LocationSummary* locations = instruction_->GetLocations();
217    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
218
219    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
220    __ Bind(GetEntryLabel());
221    SaveLiveRegisters(codegen, locations);
222
223    InvokeRuntimeCallingConvention calling_convention;
224    __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex()));
225    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pResolveString),
226                               instruction_,
227                               instruction_->GetDexPc(),
228                               this);
229    CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
230    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
231    RestoreLiveRegisters(codegen, locations);
232
233    __ jmp(GetExitLabel());
234  }
235
236  const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathX86"; }
237
238 private:
239  HLoadString* const instruction_;
240
241  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
242};
243
244class LoadClassSlowPathX86 : public SlowPathCode {
245 public:
246  LoadClassSlowPathX86(HLoadClass* cls,
247                       HInstruction* at,
248                       uint32_t dex_pc,
249                       bool do_clinit)
250      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
251    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
252  }
253
254  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
255    LocationSummary* locations = at_->GetLocations();
256    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
257    __ Bind(GetEntryLabel());
258    SaveLiveRegisters(codegen, locations);
259
260    InvokeRuntimeCallingConvention calling_convention;
261    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
262    x86_codegen->InvokeRuntime(do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
263                                          : QUICK_ENTRY_POINT(pInitializeType),
264                               at_, dex_pc_, this);
265    if (do_clinit_) {
266      CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>();
267    } else {
268      CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>();
269    }
270
271    // Move the class to the desired location.
272    Location out = locations->Out();
273    if (out.IsValid()) {
274      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
275      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
276    }
277
278    RestoreLiveRegisters(codegen, locations);
279    __ jmp(GetExitLabel());
280  }
281
282  const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathX86"; }
283
284 private:
285  // The class this slow path will load.
286  HLoadClass* const cls_;
287
288  // The instruction where this slow path is happening.
289  // (Might be the load class or an initialization check).
290  HInstruction* const at_;
291
292  // The dex PC of `at_`.
293  const uint32_t dex_pc_;
294
295  // Whether to initialize the class.
296  const bool do_clinit_;
297
298  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
299};
300
301class TypeCheckSlowPathX86 : public SlowPathCode {
302 public:
303  TypeCheckSlowPathX86(HInstruction* instruction, bool is_fatal)
304      : instruction_(instruction), is_fatal_(is_fatal) {}
305
306  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
307    LocationSummary* locations = instruction_->GetLocations();
308    Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0)
309                                                        : locations->Out();
310    DCHECK(instruction_->IsCheckCast()
311           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
312
313    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
314    __ Bind(GetEntryLabel());
315
316    if (!is_fatal_) {
317      SaveLiveRegisters(codegen, locations);
318    }
319
320    // We're moving two locations to locations that could overlap, so we need a parallel
321    // move resolver.
322    InvokeRuntimeCallingConvention calling_convention;
323    x86_codegen->EmitParallelMoves(
324        locations->InAt(1),
325        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
326        Primitive::kPrimNot,
327        object_class,
328        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
329        Primitive::kPrimNot);
330
331    if (instruction_->IsInstanceOf()) {
332      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
333                                 instruction_,
334                                 instruction_->GetDexPc(),
335                                 this);
336      CheckEntrypointTypes<
337          kQuickInstanceofNonTrivial, uint32_t, const mirror::Class*, const mirror::Class*>();
338    } else {
339      DCHECK(instruction_->IsCheckCast());
340      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
341                                 instruction_,
342                                 instruction_->GetDexPc(),
343                                 this);
344      CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
345    }
346
347    if (!is_fatal_) {
348      if (instruction_->IsInstanceOf()) {
349        x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
350      }
351      RestoreLiveRegisters(codegen, locations);
352
353      __ jmp(GetExitLabel());
354    }
355  }
356
357  const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathX86"; }
358  bool IsFatal() const OVERRIDE { return is_fatal_; }
359
360 private:
361  HInstruction* const instruction_;
362  const bool is_fatal_;
363
364  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
365};
366
367class DeoptimizationSlowPathX86 : public SlowPathCode {
368 public:
369  explicit DeoptimizationSlowPathX86(HInstruction* instruction)
370    : instruction_(instruction) {}
371
372  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
373    DCHECK(instruction_->IsDeoptimize());
374    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
375    __ Bind(GetEntryLabel());
376    SaveLiveRegisters(codegen, instruction_->GetLocations());
377    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize),
378                               instruction_,
379                               instruction_->GetDexPc(),
380                               this);
381    CheckEntrypointTypes<kQuickDeoptimize, void, void>();
382  }
383
384  const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathX86"; }
385
386 private:
387  HInstruction* const instruction_;
388  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86);
389};
390
391class ArraySetSlowPathX86 : public SlowPathCode {
392 public:
393  explicit ArraySetSlowPathX86(HInstruction* instruction) : instruction_(instruction) {}
394
395  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
396    LocationSummary* locations = instruction_->GetLocations();
397    __ Bind(GetEntryLabel());
398    SaveLiveRegisters(codegen, locations);
399
400    InvokeRuntimeCallingConvention calling_convention;
401    HParallelMove parallel_move(codegen->GetGraph()->GetArena());
402    parallel_move.AddMove(
403        locations->InAt(0),
404        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
405        Primitive::kPrimNot,
406        nullptr);
407    parallel_move.AddMove(
408        locations->InAt(1),
409        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
410        Primitive::kPrimInt,
411        nullptr);
412    parallel_move.AddMove(
413        locations->InAt(2),
414        Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
415        Primitive::kPrimNot,
416        nullptr);
417    codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
418
419    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
420    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
421                               instruction_,
422                               instruction_->GetDexPc(),
423                               this);
424    CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>();
425    RestoreLiveRegisters(codegen, locations);
426    __ jmp(GetExitLabel());
427  }
428
429  const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathX86"; }
430
431 private:
432  HInstruction* const instruction_;
433
434  DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathX86);
435};
436
437// Slow path generating a read barrier for a heap reference.
438class ReadBarrierForHeapReferenceSlowPathX86 : public SlowPathCode {
439 public:
440  ReadBarrierForHeapReferenceSlowPathX86(HInstruction* instruction,
441                                         Location out,
442                                         Location ref,
443                                         Location obj,
444                                         uint32_t offset,
445                                         Location index)
446      : instruction_(instruction),
447        out_(out),
448        ref_(ref),
449        obj_(obj),
450        offset_(offset),
451        index_(index) {
452    DCHECK(kEmitCompilerReadBarrier);
453    // If `obj` is equal to `out` or `ref`, it means the initial object
454    // has been overwritten by (or after) the heap object reference load
455    // to be instrumented, e.g.:
456    //
457    //   __ movl(out, Address(out, offset));
458    //   codegen_->GenerateReadBarrier(instruction, out_loc, out_loc, out_loc, offset);
459    //
460    // In that case, we have lost the information about the original
461    // object, and the emitted read barrier cannot work properly.
462    DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out;
463    DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref;
464  }
465
466  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
467    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
468    LocationSummary* locations = instruction_->GetLocations();
469    Register reg_out = out_.AsRegister<Register>();
470    DCHECK(locations->CanCall());
471    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
472    DCHECK(!instruction_->IsInvoke() ||
473           (instruction_->IsInvokeStaticOrDirect() &&
474            instruction_->GetLocations()->Intrinsified()));
475
476    __ Bind(GetEntryLabel());
477    SaveLiveRegisters(codegen, locations);
478
479    // We may have to change the index's value, but as `index_` is a
480    // constant member (like other "inputs" of this slow path),
481    // introduce a copy of it, `index`.
482    Location index = index_;
483    if (index_.IsValid()) {
484      // Handle `index_` for HArrayGet and intrinsic UnsafeGetObject.
485      if (instruction_->IsArrayGet()) {
486        // Compute the actual memory offset and store it in `index`.
487        Register index_reg = index_.AsRegister<Register>();
488        DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg));
489        if (codegen->IsCoreCalleeSaveRegister(index_reg)) {
490          // We are about to change the value of `index_reg` (see the
491          // calls to art::x86::X86Assembler::shll and
492          // art::x86::X86Assembler::AddImmediate below), but it has
493          // not been saved by the previous call to
494          // art::SlowPathCode::SaveLiveRegisters, as it is a
495          // callee-save register --
496          // art::SlowPathCode::SaveLiveRegisters does not consider
497          // callee-save registers, as it has been designed with the
498          // assumption that callee-save registers are supposed to be
499          // handled by the called function.  So, as a callee-save
500          // register, `index_reg` _would_ eventually be saved onto
501          // the stack, but it would be too late: we would have
502          // changed its value earlier.  Therefore, we manually save
503          // it here into another freely available register,
504          // `free_reg`, chosen of course among the caller-save
505          // registers (as a callee-save `free_reg` register would
506          // exhibit the same problem).
507          //
508          // Note we could have requested a temporary register from
509          // the register allocator instead; but we prefer not to, as
510          // this is a slow path, and we know we can find a
511          // caller-save register that is available.
512          Register free_reg = FindAvailableCallerSaveRegister(codegen);
513          __ movl(free_reg, index_reg);
514          index_reg = free_reg;
515          index = Location::RegisterLocation(index_reg);
516        } else {
517          // The initial register stored in `index_` has already been
518          // saved in the call to art::SlowPathCode::SaveLiveRegisters
519          // (as it is not a callee-save register), so we can freely
520          // use it.
521        }
522        // Shifting the index value contained in `index_reg` by the scale
523        // factor (2) cannot overflow in practice, as the runtime is
524        // unable to allocate object arrays with a size larger than
525        // 2^26 - 1 (that is, 2^28 - 4 bytes).
526        __ shll(index_reg, Immediate(TIMES_4));
527        static_assert(
528            sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
529            "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
530        __ AddImmediate(index_reg, Immediate(offset_));
531      } else {
532        DCHECK(instruction_->IsInvoke());
533        DCHECK(instruction_->GetLocations()->Intrinsified());
534        DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) ||
535               (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile))
536            << instruction_->AsInvoke()->GetIntrinsic();
537        DCHECK_EQ(offset_, 0U);
538        DCHECK(index_.IsRegisterPair());
539        // UnsafeGet's offset location is a register pair, the low
540        // part contains the correct offset.
541        index = index_.ToLow();
542      }
543    }
544
545    // We're moving two or three locations to locations that could
546    // overlap, so we need a parallel move resolver.
547    InvokeRuntimeCallingConvention calling_convention;
548    HParallelMove parallel_move(codegen->GetGraph()->GetArena());
549    parallel_move.AddMove(ref_,
550                          Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
551                          Primitive::kPrimNot,
552                          nullptr);
553    parallel_move.AddMove(obj_,
554                          Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
555                          Primitive::kPrimNot,
556                          nullptr);
557    if (index.IsValid()) {
558      parallel_move.AddMove(index,
559                            Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
560                            Primitive::kPrimInt,
561                            nullptr);
562      codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
563    } else {
564      codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
565      __ movl(calling_convention.GetRegisterAt(2), Immediate(offset_));
566    }
567    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierSlow),
568                               instruction_,
569                               instruction_->GetDexPc(),
570                               this);
571    CheckEntrypointTypes<
572        kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>();
573    x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
574
575    RestoreLiveRegisters(codegen, locations);
576    __ jmp(GetExitLabel());
577  }
578
579  const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathX86"; }
580
581 private:
582  Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) {
583    size_t ref = static_cast<int>(ref_.AsRegister<Register>());
584    size_t obj = static_cast<int>(obj_.AsRegister<Register>());
585    for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
586      if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) {
587        return static_cast<Register>(i);
588      }
589    }
590    // We shall never fail to find a free caller-save register, as
591    // there are more than two core caller-save registers on x86
592    // (meaning it is possible to find one which is different from
593    // `ref` and `obj`).
594    DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u);
595    LOG(FATAL) << "Could not find a free caller-save register";
596    UNREACHABLE();
597  }
598
599  HInstruction* const instruction_;
600  const Location out_;
601  const Location ref_;
602  const Location obj_;
603  const uint32_t offset_;
604  // An additional location containing an index to an array.
605  // Only used for HArrayGet and the UnsafeGetObject &
606  // UnsafeGetObjectVolatile intrinsics.
607  const Location index_;
608
609  DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathX86);
610};
611
612// Slow path generating a read barrier for a GC root.
613class ReadBarrierForRootSlowPathX86 : public SlowPathCode {
614 public:
615  ReadBarrierForRootSlowPathX86(HInstruction* instruction, Location out, Location root)
616      : instruction_(instruction), out_(out), root_(root) {}
617
618  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
619    LocationSummary* locations = instruction_->GetLocations();
620    Register reg_out = out_.AsRegister<Register>();
621    DCHECK(locations->CanCall());
622    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
623    DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString());
624
625    __ Bind(GetEntryLabel());
626    SaveLiveRegisters(codegen, locations);
627
628    InvokeRuntimeCallingConvention calling_convention;
629    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
630    x86_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_);
631    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierForRootSlow),
632                               instruction_,
633                               instruction_->GetDexPc(),
634                               this);
635    CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>();
636    x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
637
638    RestoreLiveRegisters(codegen, locations);
639    __ jmp(GetExitLabel());
640  }
641
642  const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathX86"; }
643
644 private:
645  HInstruction* const instruction_;
646  const Location out_;
647  const Location root_;
648
649  DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathX86);
650};
651
652#undef __
653#define __ down_cast<X86Assembler*>(GetAssembler())->
654
655inline Condition X86Condition(IfCondition cond) {
656  switch (cond) {
657    case kCondEQ: return kEqual;
658    case kCondNE: return kNotEqual;
659    case kCondLT: return kLess;
660    case kCondLE: return kLessEqual;
661    case kCondGT: return kGreater;
662    case kCondGE: return kGreaterEqual;
663    case kCondB:  return kBelow;
664    case kCondBE: return kBelowEqual;
665    case kCondA:  return kAbove;
666    case kCondAE: return kAboveEqual;
667  }
668  LOG(FATAL) << "Unreachable";
669  UNREACHABLE();
670}
671
672// Maps signed condition to unsigned condition and FP condition to x86 name.
673inline Condition X86UnsignedOrFPCondition(IfCondition cond) {
674  switch (cond) {
675    case kCondEQ: return kEqual;
676    case kCondNE: return kNotEqual;
677    // Signed to unsigned, and FP to x86 name.
678    case kCondLT: return kBelow;
679    case kCondLE: return kBelowEqual;
680    case kCondGT: return kAbove;
681    case kCondGE: return kAboveEqual;
682    // Unsigned remain unchanged.
683    case kCondB:  return kBelow;
684    case kCondBE: return kBelowEqual;
685    case kCondA:  return kAbove;
686    case kCondAE: return kAboveEqual;
687  }
688  LOG(FATAL) << "Unreachable";
689  UNREACHABLE();
690}
691
692void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
693  stream << Register(reg);
694}
695
696void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
697  stream << XmmRegister(reg);
698}
699
700size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
701  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
702  return kX86WordSize;
703}
704
705size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
706  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
707  return kX86WordSize;
708}
709
710size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
711  __ movsd(Address(ESP, stack_index), XmmRegister(reg_id));
712  return GetFloatingPointSpillSlotSize();
713}
714
715size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
716  __ movsd(XmmRegister(reg_id), Address(ESP, stack_index));
717  return GetFloatingPointSpillSlotSize();
718}
719
720void CodeGeneratorX86::InvokeRuntime(QuickEntrypointEnum entrypoint,
721                                     HInstruction* instruction,
722                                     uint32_t dex_pc,
723                                     SlowPathCode* slow_path) {
724  InvokeRuntime(GetThreadOffset<kX86WordSize>(entrypoint).Int32Value(),
725                instruction,
726                dex_pc,
727                slow_path);
728}
729
730void CodeGeneratorX86::InvokeRuntime(int32_t entry_point_offset,
731                                     HInstruction* instruction,
732                                     uint32_t dex_pc,
733                                     SlowPathCode* slow_path) {
734  ValidateInvokeRuntime(instruction, slow_path);
735  __ fs()->call(Address::Absolute(entry_point_offset));
736  RecordPcInfo(instruction, dex_pc, slow_path);
737}
738
739CodeGeneratorX86::CodeGeneratorX86(HGraph* graph,
740                                   const X86InstructionSetFeatures& isa_features,
741                                   const CompilerOptions& compiler_options,
742                                   OptimizingCompilerStats* stats)
743    : CodeGenerator(graph,
744                    kNumberOfCpuRegisters,
745                    kNumberOfXmmRegisters,
746                    kNumberOfRegisterPairs,
747                    ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
748                                        arraysize(kCoreCalleeSaves))
749                        | (1 << kFakeReturnRegister),
750                    0,
751                    compiler_options,
752                    stats),
753      block_labels_(nullptr),
754      location_builder_(graph, this),
755      instruction_visitor_(graph, this),
756      move_resolver_(graph->GetArena(), this),
757      isa_features_(isa_features),
758      method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
759      relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
760      pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
761      fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
762  // Use a fake return address register to mimic Quick.
763  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
764}
765
766Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const {
767  switch (type) {
768    case Primitive::kPrimLong: {
769      size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs);
770      X86ManagedRegister pair =
771          X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg));
772      DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]);
773      DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]);
774      blocked_core_registers_[pair.AsRegisterPairLow()] = true;
775      blocked_core_registers_[pair.AsRegisterPairHigh()] = true;
776      UpdateBlockedPairRegisters();
777      return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
778    }
779
780    case Primitive::kPrimByte:
781    case Primitive::kPrimBoolean:
782    case Primitive::kPrimChar:
783    case Primitive::kPrimShort:
784    case Primitive::kPrimInt:
785    case Primitive::kPrimNot: {
786      Register reg = static_cast<Register>(
787          FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters));
788      // Block all register pairs that contain `reg`.
789      for (int i = 0; i < kNumberOfRegisterPairs; i++) {
790        X86ManagedRegister current =
791            X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
792        if (current.AsRegisterPairLow() == reg || current.AsRegisterPairHigh() == reg) {
793          blocked_register_pairs_[i] = true;
794        }
795      }
796      return Location::RegisterLocation(reg);
797    }
798
799    case Primitive::kPrimFloat:
800    case Primitive::kPrimDouble: {
801      return Location::FpuRegisterLocation(
802          FindFreeEntry(blocked_fpu_registers_, kNumberOfXmmRegisters));
803    }
804
805    case Primitive::kPrimVoid:
806      LOG(FATAL) << "Unreachable type " << type;
807  }
808
809  return Location::NoLocation();
810}
811
812void CodeGeneratorX86::SetupBlockedRegisters(bool is_baseline) const {
813  // Don't allocate the dalvik style register pair passing.
814  blocked_register_pairs_[ECX_EDX] = true;
815
816  // Stack register is always reserved.
817  blocked_core_registers_[ESP] = true;
818
819  if (is_baseline) {
820    blocked_core_registers_[EBP] = true;
821    blocked_core_registers_[ESI] = true;
822    blocked_core_registers_[EDI] = true;
823  }
824
825  UpdateBlockedPairRegisters();
826}
827
828void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
829  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
830    X86ManagedRegister current =
831        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
832    if (blocked_core_registers_[current.AsRegisterPairLow()]
833        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
834      blocked_register_pairs_[i] = true;
835    }
836  }
837}
838
839InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
840      : HGraphVisitor(graph),
841        assembler_(codegen->GetAssembler()),
842        codegen_(codegen) {}
843
844static dwarf::Reg DWARFReg(Register reg) {
845  return dwarf::Reg::X86Core(static_cast<int>(reg));
846}
847
848void CodeGeneratorX86::GenerateFrameEntry() {
849  __ cfi().SetCurrentCFAOffset(kX86WordSize);  // return address
850  __ Bind(&frame_entry_label_);
851  bool skip_overflow_check =
852      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
853  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
854
855  if (!skip_overflow_check) {
856    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
857    RecordPcInfo(nullptr, 0);
858  }
859
860  if (HasEmptyFrame()) {
861    return;
862  }
863
864  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
865    Register reg = kCoreCalleeSaves[i];
866    if (allocated_registers_.ContainsCoreRegister(reg)) {
867      __ pushl(reg);
868      __ cfi().AdjustCFAOffset(kX86WordSize);
869      __ cfi().RelOffset(DWARFReg(reg), 0);
870    }
871  }
872
873  int adjust = GetFrameSize() - FrameEntrySpillSize();
874  __ subl(ESP, Immediate(adjust));
875  __ cfi().AdjustCFAOffset(adjust);
876  __ movl(Address(ESP, kCurrentMethodStackOffset), kMethodRegisterArgument);
877}
878
879void CodeGeneratorX86::GenerateFrameExit() {
880  __ cfi().RememberState();
881  if (!HasEmptyFrame()) {
882    int adjust = GetFrameSize() - FrameEntrySpillSize();
883    __ addl(ESP, Immediate(adjust));
884    __ cfi().AdjustCFAOffset(-adjust);
885
886    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
887      Register reg = kCoreCalleeSaves[i];
888      if (allocated_registers_.ContainsCoreRegister(reg)) {
889        __ popl(reg);
890        __ cfi().AdjustCFAOffset(-static_cast<int>(kX86WordSize));
891        __ cfi().Restore(DWARFReg(reg));
892      }
893    }
894  }
895  __ ret();
896  __ cfi().RestoreState();
897  __ cfi().DefCFAOffset(GetFrameSize());
898}
899
900void CodeGeneratorX86::Bind(HBasicBlock* block) {
901  __ Bind(GetLabelOf(block));
902}
903
904Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
905  switch (load->GetType()) {
906    case Primitive::kPrimLong:
907    case Primitive::kPrimDouble:
908      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
909
910    case Primitive::kPrimInt:
911    case Primitive::kPrimNot:
912    case Primitive::kPrimFloat:
913      return Location::StackSlot(GetStackSlot(load->GetLocal()));
914
915    case Primitive::kPrimBoolean:
916    case Primitive::kPrimByte:
917    case Primitive::kPrimChar:
918    case Primitive::kPrimShort:
919    case Primitive::kPrimVoid:
920      LOG(FATAL) << "Unexpected type " << load->GetType();
921      UNREACHABLE();
922  }
923
924  LOG(FATAL) << "Unreachable";
925  UNREACHABLE();
926}
927
928Location InvokeDexCallingConventionVisitorX86::GetReturnLocation(Primitive::Type type) const {
929  switch (type) {
930    case Primitive::kPrimBoolean:
931    case Primitive::kPrimByte:
932    case Primitive::kPrimChar:
933    case Primitive::kPrimShort:
934    case Primitive::kPrimInt:
935    case Primitive::kPrimNot:
936      return Location::RegisterLocation(EAX);
937
938    case Primitive::kPrimLong:
939      return Location::RegisterPairLocation(EAX, EDX);
940
941    case Primitive::kPrimVoid:
942      return Location::NoLocation();
943
944    case Primitive::kPrimDouble:
945    case Primitive::kPrimFloat:
946      return Location::FpuRegisterLocation(XMM0);
947  }
948
949  UNREACHABLE();
950}
951
952Location InvokeDexCallingConventionVisitorX86::GetMethodLocation() const {
953  return Location::RegisterLocation(kMethodRegisterArgument);
954}
955
956Location InvokeDexCallingConventionVisitorX86::GetNextLocation(Primitive::Type type) {
957  switch (type) {
958    case Primitive::kPrimBoolean:
959    case Primitive::kPrimByte:
960    case Primitive::kPrimChar:
961    case Primitive::kPrimShort:
962    case Primitive::kPrimInt:
963    case Primitive::kPrimNot: {
964      uint32_t index = gp_index_++;
965      stack_index_++;
966      if (index < calling_convention.GetNumberOfRegisters()) {
967        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
968      } else {
969        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
970      }
971    }
972
973    case Primitive::kPrimLong: {
974      uint32_t index = gp_index_;
975      gp_index_ += 2;
976      stack_index_ += 2;
977      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
978        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
979            calling_convention.GetRegisterPairAt(index));
980        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
981      } else {
982        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
983      }
984    }
985
986    case Primitive::kPrimFloat: {
987      uint32_t index = float_index_++;
988      stack_index_++;
989      if (index < calling_convention.GetNumberOfFpuRegisters()) {
990        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
991      } else {
992        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
993      }
994    }
995
996    case Primitive::kPrimDouble: {
997      uint32_t index = float_index_++;
998      stack_index_ += 2;
999      if (index < calling_convention.GetNumberOfFpuRegisters()) {
1000        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
1001      } else {
1002        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
1003      }
1004    }
1005
1006    case Primitive::kPrimVoid:
1007      LOG(FATAL) << "Unexpected parameter type " << type;
1008      break;
1009  }
1010  return Location::NoLocation();
1011}
1012
1013void CodeGeneratorX86::Move32(Location destination, Location source) {
1014  if (source.Equals(destination)) {
1015    return;
1016  }
1017  if (destination.IsRegister()) {
1018    if (source.IsRegister()) {
1019      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
1020    } else if (source.IsFpuRegister()) {
1021      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
1022    } else {
1023      DCHECK(source.IsStackSlot());
1024      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
1025    }
1026  } else if (destination.IsFpuRegister()) {
1027    if (source.IsRegister()) {
1028      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
1029    } else if (source.IsFpuRegister()) {
1030      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
1031    } else {
1032      DCHECK(source.IsStackSlot());
1033      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
1034    }
1035  } else {
1036    DCHECK(destination.IsStackSlot()) << destination;
1037    if (source.IsRegister()) {
1038      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
1039    } else if (source.IsFpuRegister()) {
1040      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
1041    } else if (source.IsConstant()) {
1042      HConstant* constant = source.GetConstant();
1043      int32_t value = GetInt32ValueOf(constant);
1044      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
1045    } else {
1046      DCHECK(source.IsStackSlot());
1047      __ pushl(Address(ESP, source.GetStackIndex()));
1048      __ popl(Address(ESP, destination.GetStackIndex()));
1049    }
1050  }
1051}
1052
1053void CodeGeneratorX86::Move64(Location destination, Location source) {
1054  if (source.Equals(destination)) {
1055    return;
1056  }
1057  if (destination.IsRegisterPair()) {
1058    if (source.IsRegisterPair()) {
1059      EmitParallelMoves(
1060          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
1061          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
1062          Primitive::kPrimInt,
1063          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
1064          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
1065          Primitive::kPrimInt);
1066    } else if (source.IsFpuRegister()) {
1067      XmmRegister src_reg = source.AsFpuRegister<XmmRegister>();
1068      __ movd(destination.AsRegisterPairLow<Register>(), src_reg);
1069      __ psrlq(src_reg, Immediate(32));
1070      __ movd(destination.AsRegisterPairHigh<Register>(), src_reg);
1071    } else {
1072      // No conflict possible, so just do the moves.
1073      DCHECK(source.IsDoubleStackSlot());
1074      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
1075      __ movl(destination.AsRegisterPairHigh<Register>(),
1076              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
1077    }
1078  } else if (destination.IsFpuRegister()) {
1079    if (source.IsFpuRegister()) {
1080      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
1081    } else if (source.IsDoubleStackSlot()) {
1082      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
1083    } else if (source.IsRegisterPair()) {
1084      size_t elem_size = Primitive::ComponentSize(Primitive::kPrimInt);
1085      // Create stack space for 2 elements.
1086      __ subl(ESP, Immediate(2 * elem_size));
1087      __ movl(Address(ESP, 0), source.AsRegisterPairLow<Register>());
1088      __ movl(Address(ESP, elem_size), source.AsRegisterPairHigh<Register>());
1089      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
1090      // And remove the temporary stack space we allocated.
1091      __ addl(ESP, Immediate(2 * elem_size));
1092    } else {
1093      LOG(FATAL) << "Unimplemented";
1094    }
1095  } else {
1096    DCHECK(destination.IsDoubleStackSlot()) << destination;
1097    if (source.IsRegisterPair()) {
1098      // No conflict possible, so just do the moves.
1099      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
1100      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
1101              source.AsRegisterPairHigh<Register>());
1102    } else if (source.IsFpuRegister()) {
1103      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
1104    } else if (source.IsConstant()) {
1105      HConstant* constant = source.GetConstant();
1106      int64_t value;
1107      if (constant->IsLongConstant()) {
1108        value = constant->AsLongConstant()->GetValue();
1109      } else {
1110        DCHECK(constant->IsDoubleConstant());
1111        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
1112      }
1113      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(Low32Bits(value)));
1114      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value)));
1115    } else {
1116      DCHECK(source.IsDoubleStackSlot()) << source;
1117      EmitParallelMoves(
1118          Location::StackSlot(source.GetStackIndex()),
1119          Location::StackSlot(destination.GetStackIndex()),
1120          Primitive::kPrimInt,
1121          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
1122          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)),
1123          Primitive::kPrimInt);
1124    }
1125  }
1126}
1127
1128void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstruction* move_for) {
1129  LocationSummary* locations = instruction->GetLocations();
1130  if (instruction->IsCurrentMethod()) {
1131    Move32(location, Location::StackSlot(kCurrentMethodStackOffset));
1132  } else if (locations != nullptr && locations->Out().Equals(location)) {
1133    return;
1134  } else if (locations != nullptr && locations->Out().IsConstant()) {
1135    HConstant* const_to_move = locations->Out().GetConstant();
1136    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
1137      Immediate imm(GetInt32ValueOf(const_to_move));
1138      if (location.IsRegister()) {
1139        __ movl(location.AsRegister<Register>(), imm);
1140      } else if (location.IsStackSlot()) {
1141        __ movl(Address(ESP, location.GetStackIndex()), imm);
1142      } else {
1143        DCHECK(location.IsConstant());
1144        DCHECK_EQ(location.GetConstant(), const_to_move);
1145      }
1146    } else if (const_to_move->IsLongConstant()) {
1147      int64_t value = const_to_move->AsLongConstant()->GetValue();
1148      if (location.IsRegisterPair()) {
1149        __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
1150        __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
1151      } else if (location.IsDoubleStackSlot()) {
1152        __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value)));
1153        __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)),
1154                Immediate(High32Bits(value)));
1155      } else {
1156        DCHECK(location.IsConstant());
1157        DCHECK_EQ(location.GetConstant(), instruction);
1158      }
1159    }
1160  } else if (instruction->IsTemporary()) {
1161    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
1162    if (temp_location.IsStackSlot()) {
1163      Move32(location, temp_location);
1164    } else {
1165      DCHECK(temp_location.IsDoubleStackSlot());
1166      Move64(location, temp_location);
1167    }
1168  } else if (instruction->IsLoadLocal()) {
1169    int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
1170    switch (instruction->GetType()) {
1171      case Primitive::kPrimBoolean:
1172      case Primitive::kPrimByte:
1173      case Primitive::kPrimChar:
1174      case Primitive::kPrimShort:
1175      case Primitive::kPrimInt:
1176      case Primitive::kPrimNot:
1177      case Primitive::kPrimFloat:
1178        Move32(location, Location::StackSlot(slot));
1179        break;
1180
1181      case Primitive::kPrimLong:
1182      case Primitive::kPrimDouble:
1183        Move64(location, Location::DoubleStackSlot(slot));
1184        break;
1185
1186      default:
1187        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
1188    }
1189  } else {
1190    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
1191    switch (instruction->GetType()) {
1192      case Primitive::kPrimBoolean:
1193      case Primitive::kPrimByte:
1194      case Primitive::kPrimChar:
1195      case Primitive::kPrimShort:
1196      case Primitive::kPrimInt:
1197      case Primitive::kPrimNot:
1198      case Primitive::kPrimFloat:
1199        Move32(location, locations->Out());
1200        break;
1201
1202      case Primitive::kPrimLong:
1203      case Primitive::kPrimDouble:
1204        Move64(location, locations->Out());
1205        break;
1206
1207      default:
1208        LOG(FATAL) << "Unexpected type " << instruction->GetType();
1209    }
1210  }
1211}
1212
1213void CodeGeneratorX86::MoveConstant(Location location, int32_t value) {
1214  DCHECK(location.IsRegister());
1215  __ movl(location.AsRegister<Register>(), Immediate(value));
1216}
1217
1218void CodeGeneratorX86::MoveLocation(Location dst, Location src, Primitive::Type dst_type) {
1219  if (Primitive::Is64BitType(dst_type)) {
1220    Move64(dst, src);
1221  } else {
1222    Move32(dst, src);
1223  }
1224}
1225
1226void CodeGeneratorX86::AddLocationAsTemp(Location location, LocationSummary* locations) {
1227  if (location.IsRegister()) {
1228    locations->AddTemp(location);
1229  } else if (location.IsRegisterPair()) {
1230    locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>()));
1231    locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>()));
1232  } else {
1233    UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location;
1234  }
1235}
1236
1237void InstructionCodeGeneratorX86::HandleGoto(HInstruction* got, HBasicBlock* successor) {
1238  DCHECK(!successor->IsExitBlock());
1239
1240  HBasicBlock* block = got->GetBlock();
1241  HInstruction* previous = got->GetPrevious();
1242
1243  HLoopInformation* info = block->GetLoopInformation();
1244  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
1245    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
1246    return;
1247  }
1248
1249  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
1250    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
1251  }
1252  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
1253    __ jmp(codegen_->GetLabelOf(successor));
1254  }
1255}
1256
1257void LocationsBuilderX86::VisitGoto(HGoto* got) {
1258  got->SetLocations(nullptr);
1259}
1260
1261void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
1262  HandleGoto(got, got->GetSuccessor());
1263}
1264
1265void LocationsBuilderX86::VisitTryBoundary(HTryBoundary* try_boundary) {
1266  try_boundary->SetLocations(nullptr);
1267}
1268
1269void InstructionCodeGeneratorX86::VisitTryBoundary(HTryBoundary* try_boundary) {
1270  HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
1271  if (!successor->IsExitBlock()) {
1272    HandleGoto(try_boundary, successor);
1273  }
1274}
1275
1276void LocationsBuilderX86::VisitExit(HExit* exit) {
1277  exit->SetLocations(nullptr);
1278}
1279
1280void InstructionCodeGeneratorX86::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
1281}
1282
1283void InstructionCodeGeneratorX86::GenerateFPJumps(HCondition* cond,
1284                                                  Label* true_label,
1285                                                  Label* false_label) {
1286  if (cond->IsFPConditionTrueIfNaN()) {
1287    __ j(kUnordered, true_label);
1288  } else if (cond->IsFPConditionFalseIfNaN()) {
1289    __ j(kUnordered, false_label);
1290  }
1291  __ j(X86UnsignedOrFPCondition(cond->GetCondition()), true_label);
1292}
1293
1294void InstructionCodeGeneratorX86::GenerateLongComparesAndJumps(HCondition* cond,
1295                                                               Label* true_label,
1296                                                               Label* false_label) {
1297  LocationSummary* locations = cond->GetLocations();
1298  Location left = locations->InAt(0);
1299  Location right = locations->InAt(1);
1300  IfCondition if_cond = cond->GetCondition();
1301
1302  Register left_high = left.AsRegisterPairHigh<Register>();
1303  Register left_low = left.AsRegisterPairLow<Register>();
1304  IfCondition true_high_cond = if_cond;
1305  IfCondition false_high_cond = cond->GetOppositeCondition();
1306  Condition final_condition = X86UnsignedOrFPCondition(if_cond);  // unsigned on lower part
1307
1308  // Set the conditions for the test, remembering that == needs to be
1309  // decided using the low words.
1310  switch (if_cond) {
1311    case kCondEQ:
1312    case kCondNE:
1313      // Nothing to do.
1314      break;
1315    case kCondLT:
1316      false_high_cond = kCondGT;
1317      break;
1318    case kCondLE:
1319      true_high_cond = kCondLT;
1320      break;
1321    case kCondGT:
1322      false_high_cond = kCondLT;
1323      break;
1324    case kCondGE:
1325      true_high_cond = kCondGT;
1326      break;
1327    case kCondB:
1328      false_high_cond = kCondA;
1329      break;
1330    case kCondBE:
1331      true_high_cond = kCondB;
1332      break;
1333    case kCondA:
1334      false_high_cond = kCondB;
1335      break;
1336    case kCondAE:
1337      true_high_cond = kCondA;
1338      break;
1339  }
1340
1341  if (right.IsConstant()) {
1342    int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
1343    int32_t val_high = High32Bits(value);
1344    int32_t val_low = Low32Bits(value);
1345
1346    if (val_high == 0) {
1347      __ testl(left_high, left_high);
1348    } else {
1349      __ cmpl(left_high, Immediate(val_high));
1350    }
1351    if (if_cond == kCondNE) {
1352      __ j(X86Condition(true_high_cond), true_label);
1353    } else if (if_cond == kCondEQ) {
1354      __ j(X86Condition(false_high_cond), false_label);
1355    } else {
1356      __ j(X86Condition(true_high_cond), true_label);
1357      __ j(X86Condition(false_high_cond), false_label);
1358    }
1359    // Must be equal high, so compare the lows.
1360    if (val_low == 0) {
1361      __ testl(left_low, left_low);
1362    } else {
1363      __ cmpl(left_low, Immediate(val_low));
1364    }
1365  } else {
1366    Register right_high = right.AsRegisterPairHigh<Register>();
1367    Register right_low = right.AsRegisterPairLow<Register>();
1368
1369    __ cmpl(left_high, right_high);
1370    if (if_cond == kCondNE) {
1371      __ j(X86Condition(true_high_cond), true_label);
1372    } else if (if_cond == kCondEQ) {
1373      __ j(X86Condition(false_high_cond), false_label);
1374    } else {
1375      __ j(X86Condition(true_high_cond), true_label);
1376      __ j(X86Condition(false_high_cond), false_label);
1377    }
1378    // Must be equal high, so compare the lows.
1379    __ cmpl(left_low, right_low);
1380  }
1381  // The last comparison might be unsigned.
1382  __ j(final_condition, true_label);
1383}
1384
1385void InstructionCodeGeneratorX86::GenerateCompareTestAndBranch(HCondition* condition,
1386                                                               Label* true_target_in,
1387                                                               Label* false_target_in) {
1388  // Generated branching requires both targets to be explicit. If either of the
1389  // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead.
1390  Label fallthrough_target;
1391  Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in;
1392  Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in;
1393
1394  LocationSummary* locations = condition->GetLocations();
1395  Location left = locations->InAt(0);
1396  Location right = locations->InAt(1);
1397
1398  Primitive::Type type = condition->InputAt(0)->GetType();
1399  switch (type) {
1400    case Primitive::kPrimLong:
1401      GenerateLongComparesAndJumps(condition, true_target, false_target);
1402      break;
1403    case Primitive::kPrimFloat:
1404      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1405      GenerateFPJumps(condition, true_target, false_target);
1406      break;
1407    case Primitive::kPrimDouble:
1408      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1409      GenerateFPJumps(condition, true_target, false_target);
1410      break;
1411    default:
1412      LOG(FATAL) << "Unexpected compare type " << type;
1413  }
1414
1415  if (false_target != &fallthrough_target) {
1416    __ jmp(false_target);
1417  }
1418
1419  if (fallthrough_target.IsLinked()) {
1420    __ Bind(&fallthrough_target);
1421  }
1422}
1423
1424static bool AreEflagsSetFrom(HInstruction* cond, HInstruction* branch) {
1425  // Moves may affect the eflags register (move zero uses xorl), so the EFLAGS
1426  // are set only strictly before `branch`. We can't use the eflags on long/FP
1427  // conditions if they are materialized due to the complex branching.
1428  return cond->IsCondition() &&
1429         cond->GetNext() == branch &&
1430         cond->InputAt(0)->GetType() != Primitive::kPrimLong &&
1431         !Primitive::IsFloatingPointType(cond->InputAt(0)->GetType());
1432}
1433
1434void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instruction,
1435                                                        size_t condition_input_index,
1436                                                        Label* true_target,
1437                                                        Label* false_target) {
1438  HInstruction* cond = instruction->InputAt(condition_input_index);
1439
1440  if (true_target == nullptr && false_target == nullptr) {
1441    // Nothing to do. The code always falls through.
1442    return;
1443  } else if (cond->IsIntConstant()) {
1444    // Constant condition, statically compared against 1.
1445    if (cond->AsIntConstant()->IsOne()) {
1446      if (true_target != nullptr) {
1447        __ jmp(true_target);
1448      }
1449    } else {
1450      DCHECK(cond->AsIntConstant()->IsZero());
1451      if (false_target != nullptr) {
1452        __ jmp(false_target);
1453      }
1454    }
1455    return;
1456  }
1457
1458  // The following code generates these patterns:
1459  //  (1) true_target == nullptr && false_target != nullptr
1460  //        - opposite condition true => branch to false_target
1461  //  (2) true_target != nullptr && false_target == nullptr
1462  //        - condition true => branch to true_target
1463  //  (3) true_target != nullptr && false_target != nullptr
1464  //        - condition true => branch to true_target
1465  //        - branch to false_target
1466  if (IsBooleanValueOrMaterializedCondition(cond)) {
1467    if (AreEflagsSetFrom(cond, instruction)) {
1468      if (true_target == nullptr) {
1469        __ j(X86Condition(cond->AsCondition()->GetOppositeCondition()), false_target);
1470      } else {
1471        __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
1472      }
1473    } else {
1474      // Materialized condition, compare against 0.
1475      Location lhs = instruction->GetLocations()->InAt(condition_input_index);
1476      if (lhs.IsRegister()) {
1477        __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1478      } else {
1479        __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
1480      }
1481      if (true_target == nullptr) {
1482        __ j(kEqual, false_target);
1483      } else {
1484        __ j(kNotEqual, true_target);
1485      }
1486    }
1487  } else {
1488    // Condition has not been materialized, use its inputs as the comparison and
1489    // its condition as the branch condition.
1490    HCondition* condition = cond->AsCondition();
1491
1492    // If this is a long or FP comparison that has been folded into
1493    // the HCondition, generate the comparison directly.
1494    Primitive::Type type = condition->InputAt(0)->GetType();
1495    if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
1496      GenerateCompareTestAndBranch(condition, true_target, false_target);
1497      return;
1498    }
1499
1500    Location lhs = condition->GetLocations()->InAt(0);
1501    Location rhs = condition->GetLocations()->InAt(1);
1502    // LHS is guaranteed to be in a register (see LocationsBuilderX86::VisitCondition).
1503    if (rhs.IsRegister()) {
1504      __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1505    } else if (rhs.IsConstant()) {
1506      int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1507      if (constant == 0) {
1508        __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1509      } else {
1510        __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1511      }
1512    } else {
1513      __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1514    }
1515    if (true_target == nullptr) {
1516      __ j(X86Condition(condition->GetOppositeCondition()), false_target);
1517    } else {
1518      __ j(X86Condition(condition->GetCondition()), true_target);
1519    }
1520  }
1521
1522  // If neither branch falls through (case 3), the conditional branch to `true_target`
1523  // was already emitted (case 2) and we need to emit a jump to `false_target`.
1524  if (true_target != nullptr && false_target != nullptr) {
1525    __ jmp(false_target);
1526  }
1527}
1528
1529void LocationsBuilderX86::VisitIf(HIf* if_instr) {
1530  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
1531  if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
1532    locations->SetInAt(0, Location::Any());
1533  }
1534}
1535
1536void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
1537  HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
1538  HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
1539  Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
1540      nullptr : codegen_->GetLabelOf(true_successor);
1541  Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
1542      nullptr : codegen_->GetLabelOf(false_successor);
1543  GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
1544}
1545
1546void LocationsBuilderX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1547  LocationSummary* locations = new (GetGraph()->GetArena())
1548      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
1549  if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
1550    locations->SetInAt(0, Location::Any());
1551  }
1552}
1553
1554void InstructionCodeGeneratorX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1555  SlowPathCode* slow_path = new (GetGraph()->GetArena())
1556      DeoptimizationSlowPathX86(deoptimize);
1557  codegen_->AddSlowPath(slow_path);
1558  GenerateTestAndBranch(deoptimize,
1559                        /* condition_input_index */ 0,
1560                        slow_path->GetEntryLabel(),
1561                        /* false_target */ nullptr);
1562}
1563
1564void LocationsBuilderX86::VisitLocal(HLocal* local) {
1565  local->SetLocations(nullptr);
1566}
1567
1568void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
1569  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
1570}
1571
1572void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
1573  local->SetLocations(nullptr);
1574}
1575
1576void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_UNUSED) {
1577  // Nothing to do, this is driven by the code generator.
1578}
1579
1580void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
1581  LocationSummary* locations =
1582      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
1583  switch (store->InputAt(1)->GetType()) {
1584    case Primitive::kPrimBoolean:
1585    case Primitive::kPrimByte:
1586    case Primitive::kPrimChar:
1587    case Primitive::kPrimShort:
1588    case Primitive::kPrimInt:
1589    case Primitive::kPrimNot:
1590    case Primitive::kPrimFloat:
1591      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
1592      break;
1593
1594    case Primitive::kPrimLong:
1595    case Primitive::kPrimDouble:
1596      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
1597      break;
1598
1599    default:
1600      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
1601  }
1602}
1603
1604void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store ATTRIBUTE_UNUSED) {
1605}
1606
1607void LocationsBuilderX86::VisitCondition(HCondition* cond) {
1608  LocationSummary* locations =
1609      new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
1610  // Handle the long/FP comparisons made in instruction simplification.
1611  switch (cond->InputAt(0)->GetType()) {
1612    case Primitive::kPrimLong: {
1613      locations->SetInAt(0, Location::RequiresRegister());
1614      locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1)));
1615      if (cond->NeedsMaterialization()) {
1616        locations->SetOut(Location::RequiresRegister());
1617      }
1618      break;
1619    }
1620    case Primitive::kPrimFloat:
1621    case Primitive::kPrimDouble: {
1622      locations->SetInAt(0, Location::RequiresFpuRegister());
1623      locations->SetInAt(1, Location::RequiresFpuRegister());
1624      if (cond->NeedsMaterialization()) {
1625        locations->SetOut(Location::RequiresRegister());
1626      }
1627      break;
1628    }
1629    default:
1630      locations->SetInAt(0, Location::RequiresRegister());
1631      locations->SetInAt(1, Location::Any());
1632      if (cond->NeedsMaterialization()) {
1633        // We need a byte register.
1634        locations->SetOut(Location::RegisterLocation(ECX));
1635      }
1636      break;
1637  }
1638}
1639
1640void InstructionCodeGeneratorX86::VisitCondition(HCondition* cond) {
1641  if (!cond->NeedsMaterialization()) {
1642    return;
1643  }
1644
1645  LocationSummary* locations = cond->GetLocations();
1646  Location lhs = locations->InAt(0);
1647  Location rhs = locations->InAt(1);
1648  Register reg = locations->Out().AsRegister<Register>();
1649  Label true_label, false_label;
1650
1651  switch (cond->InputAt(0)->GetType()) {
1652    default: {
1653      // Integer case.
1654
1655      // Clear output register: setb only sets the low byte.
1656      __ xorl(reg, reg);
1657
1658      if (rhs.IsRegister()) {
1659        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1660      } else if (rhs.IsConstant()) {
1661        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1662        if (constant == 0) {
1663          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1664        } else {
1665          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1666        }
1667      } else {
1668        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1669      }
1670      __ setb(X86Condition(cond->GetCondition()), reg);
1671      return;
1672    }
1673    case Primitive::kPrimLong:
1674      GenerateLongComparesAndJumps(cond, &true_label, &false_label);
1675      break;
1676    case Primitive::kPrimFloat:
1677      __ ucomiss(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1678      GenerateFPJumps(cond, &true_label, &false_label);
1679      break;
1680    case Primitive::kPrimDouble:
1681      __ ucomisd(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1682      GenerateFPJumps(cond, &true_label, &false_label);
1683      break;
1684  }
1685
1686  // Convert the jumps into the result.
1687  NearLabel done_label;
1688
1689  // False case: result = 0.
1690  __ Bind(&false_label);
1691  __ xorl(reg, reg);
1692  __ jmp(&done_label);
1693
1694  // True case: result = 1.
1695  __ Bind(&true_label);
1696  __ movl(reg, Immediate(1));
1697  __ Bind(&done_label);
1698}
1699
1700void LocationsBuilderX86::VisitEqual(HEqual* comp) {
1701  VisitCondition(comp);
1702}
1703
1704void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
1705  VisitCondition(comp);
1706}
1707
1708void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
1709  VisitCondition(comp);
1710}
1711
1712void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
1713  VisitCondition(comp);
1714}
1715
1716void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
1717  VisitCondition(comp);
1718}
1719
1720void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
1721  VisitCondition(comp);
1722}
1723
1724void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1725  VisitCondition(comp);
1726}
1727
1728void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1729  VisitCondition(comp);
1730}
1731
1732void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
1733  VisitCondition(comp);
1734}
1735
1736void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
1737  VisitCondition(comp);
1738}
1739
1740void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1741  VisitCondition(comp);
1742}
1743
1744void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1745  VisitCondition(comp);
1746}
1747
1748void LocationsBuilderX86::VisitBelow(HBelow* comp) {
1749  VisitCondition(comp);
1750}
1751
1752void InstructionCodeGeneratorX86::VisitBelow(HBelow* comp) {
1753  VisitCondition(comp);
1754}
1755
1756void LocationsBuilderX86::VisitBelowOrEqual(HBelowOrEqual* comp) {
1757  VisitCondition(comp);
1758}
1759
1760void InstructionCodeGeneratorX86::VisitBelowOrEqual(HBelowOrEqual* comp) {
1761  VisitCondition(comp);
1762}
1763
1764void LocationsBuilderX86::VisitAbove(HAbove* comp) {
1765  VisitCondition(comp);
1766}
1767
1768void InstructionCodeGeneratorX86::VisitAbove(HAbove* comp) {
1769  VisitCondition(comp);
1770}
1771
1772void LocationsBuilderX86::VisitAboveOrEqual(HAboveOrEqual* comp) {
1773  VisitCondition(comp);
1774}
1775
1776void InstructionCodeGeneratorX86::VisitAboveOrEqual(HAboveOrEqual* comp) {
1777  VisitCondition(comp);
1778}
1779
1780void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1781  LocationSummary* locations =
1782      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1783  locations->SetOut(Location::ConstantLocation(constant));
1784}
1785
1786void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) {
1787  // Will be generated at use site.
1788}
1789
1790void LocationsBuilderX86::VisitNullConstant(HNullConstant* constant) {
1791  LocationSummary* locations =
1792      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1793  locations->SetOut(Location::ConstantLocation(constant));
1794}
1795
1796void InstructionCodeGeneratorX86::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) {
1797  // Will be generated at use site.
1798}
1799
1800void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1801  LocationSummary* locations =
1802      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1803  locations->SetOut(Location::ConstantLocation(constant));
1804}
1805
1806void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) {
1807  // Will be generated at use site.
1808}
1809
1810void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1811  LocationSummary* locations =
1812      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1813  locations->SetOut(Location::ConstantLocation(constant));
1814}
1815
1816void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) {
1817  // Will be generated at use site.
1818}
1819
1820void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1821  LocationSummary* locations =
1822      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1823  locations->SetOut(Location::ConstantLocation(constant));
1824}
1825
1826void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) {
1827  // Will be generated at use site.
1828}
1829
1830void LocationsBuilderX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1831  memory_barrier->SetLocations(nullptr);
1832}
1833
1834void InstructionCodeGeneratorX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1835  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1836}
1837
1838void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1839  ret->SetLocations(nullptr);
1840}
1841
1842void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) {
1843  codegen_->GenerateFrameExit();
1844}
1845
1846void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1847  LocationSummary* locations =
1848      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1849  switch (ret->InputAt(0)->GetType()) {
1850    case Primitive::kPrimBoolean:
1851    case Primitive::kPrimByte:
1852    case Primitive::kPrimChar:
1853    case Primitive::kPrimShort:
1854    case Primitive::kPrimInt:
1855    case Primitive::kPrimNot:
1856      locations->SetInAt(0, Location::RegisterLocation(EAX));
1857      break;
1858
1859    case Primitive::kPrimLong:
1860      locations->SetInAt(
1861          0, Location::RegisterPairLocation(EAX, EDX));
1862      break;
1863
1864    case Primitive::kPrimFloat:
1865    case Primitive::kPrimDouble:
1866      locations->SetInAt(
1867          0, Location::FpuRegisterLocation(XMM0));
1868      break;
1869
1870    default:
1871      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1872  }
1873}
1874
1875void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1876  if (kIsDebugBuild) {
1877    switch (ret->InputAt(0)->GetType()) {
1878      case Primitive::kPrimBoolean:
1879      case Primitive::kPrimByte:
1880      case Primitive::kPrimChar:
1881      case Primitive::kPrimShort:
1882      case Primitive::kPrimInt:
1883      case Primitive::kPrimNot:
1884        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1885        break;
1886
1887      case Primitive::kPrimLong:
1888        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1889        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1890        break;
1891
1892      case Primitive::kPrimFloat:
1893      case Primitive::kPrimDouble:
1894        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1895        break;
1896
1897      default:
1898        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1899    }
1900  }
1901  codegen_->GenerateFrameExit();
1902}
1903
1904void LocationsBuilderX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1905  // The trampoline uses the same calling convention as dex calling conventions,
1906  // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain
1907  // the method_idx.
1908  HandleInvoke(invoke);
1909}
1910
1911void InstructionCodeGeneratorX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1912  codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
1913}
1914
1915void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1916  // When we do not run baseline, explicit clinit checks triggered by static
1917  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1918  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1919
1920  IntrinsicLocationsBuilderX86 intrinsic(codegen_);
1921  if (intrinsic.TryDispatch(invoke)) {
1922    if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) {
1923      invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any());
1924    }
1925    return;
1926  }
1927
1928  HandleInvoke(invoke);
1929
1930  // For PC-relative dex cache the invoke has an extra input, the PC-relative address base.
1931  if (invoke->HasPcRelativeDexCache()) {
1932    invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(),
1933                                    Location::RequiresRegister());
1934  }
1935
1936  if (codegen_->IsBaseline()) {
1937    // Baseline does not have enough registers if the current method also
1938    // needs a register. We therefore do not require a register for it, and let
1939    // the code generation of the invoke handle it.
1940    LocationSummary* locations = invoke->GetLocations();
1941    Location location = locations->InAt(invoke->GetSpecialInputIndex());
1942    if (location.IsUnallocated() && location.GetPolicy() == Location::kRequiresRegister) {
1943      locations->SetInAt(invoke->GetSpecialInputIndex(), Location::NoLocation());
1944    }
1945  }
1946}
1947
1948static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) {
1949  if (invoke->GetLocations()->Intrinsified()) {
1950    IntrinsicCodeGeneratorX86 intrinsic(codegen);
1951    intrinsic.Dispatch(invoke);
1952    return true;
1953  }
1954  return false;
1955}
1956
1957void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1958  // When we do not run baseline, explicit clinit checks triggered by static
1959  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1960  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1961
1962  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1963    return;
1964  }
1965
1966  LocationSummary* locations = invoke->GetLocations();
1967  codegen_->GenerateStaticOrDirectCall(
1968      invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
1969  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1970}
1971
1972void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1973  HandleInvoke(invoke);
1974}
1975
1976void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
1977  InvokeDexCallingConventionVisitorX86 calling_convention_visitor;
1978  CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
1979}
1980
1981void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1982  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1983    return;
1984  }
1985
1986  codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
1987  DCHECK(!codegen_->IsLeafMethod());
1988  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1989}
1990
1991void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1992  // This call to HandleInvoke allocates a temporary (core) register
1993  // which is also used to transfer the hidden argument from FP to
1994  // core register.
1995  HandleInvoke(invoke);
1996  // Add the hidden argument.
1997  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
1998}
1999
2000void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
2001  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
2002  LocationSummary* locations = invoke->GetLocations();
2003  Register temp = locations->GetTemp(0).AsRegister<Register>();
2004  XmmRegister hidden_reg = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
2005  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
2006      invoke->GetImtIndex() % mirror::Class::kImtSize, kX86PointerSize).Uint32Value();
2007  Location receiver = locations->InAt(0);
2008  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
2009
2010  // Set the hidden argument. This is safe to do this here, as XMM7
2011  // won't be modified thereafter, before the `call` instruction.
2012  DCHECK_EQ(XMM7, hidden_reg);
2013  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
2014  __ movd(hidden_reg, temp);
2015
2016  if (receiver.IsStackSlot()) {
2017    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
2018    // /* HeapReference<Class> */ temp = temp->klass_
2019    __ movl(temp, Address(temp, class_offset));
2020  } else {
2021    // /* HeapReference<Class> */ temp = receiver->klass_
2022    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
2023  }
2024  codegen_->MaybeRecordImplicitNullCheck(invoke);
2025  // Instead of simply (possibly) unpoisoning `temp` here, we should
2026  // emit a read barrier for the previous class reference load.
2027  // However this is not required in practice, as this is an
2028  // intermediate/temporary reference and because the current
2029  // concurrent copying collector keeps the from-space memory
2030  // intact/accessible until the end of the marking phase (the
2031  // concurrent copying collector may not in the future).
2032  __ MaybeUnpoisonHeapReference(temp);
2033  // temp = temp->GetImtEntryAt(method_offset);
2034  __ movl(temp, Address(temp, method_offset));
2035  // call temp->GetEntryPoint();
2036  __ call(Address(temp,
2037                  ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
2038
2039  DCHECK(!codegen_->IsLeafMethod());
2040  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2041}
2042
2043void LocationsBuilderX86::VisitNeg(HNeg* neg) {
2044  LocationSummary* locations =
2045      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
2046  switch (neg->GetResultType()) {
2047    case Primitive::kPrimInt:
2048    case Primitive::kPrimLong:
2049      locations->SetInAt(0, Location::RequiresRegister());
2050      locations->SetOut(Location::SameAsFirstInput());
2051      break;
2052
2053    case Primitive::kPrimFloat:
2054      locations->SetInAt(0, Location::RequiresFpuRegister());
2055      locations->SetOut(Location::SameAsFirstInput());
2056      locations->AddTemp(Location::RequiresRegister());
2057      locations->AddTemp(Location::RequiresFpuRegister());
2058      break;
2059
2060    case Primitive::kPrimDouble:
2061      locations->SetInAt(0, Location::RequiresFpuRegister());
2062      locations->SetOut(Location::SameAsFirstInput());
2063      locations->AddTemp(Location::RequiresFpuRegister());
2064      break;
2065
2066    default:
2067      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
2068  }
2069}
2070
2071void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
2072  LocationSummary* locations = neg->GetLocations();
2073  Location out = locations->Out();
2074  Location in = locations->InAt(0);
2075  switch (neg->GetResultType()) {
2076    case Primitive::kPrimInt:
2077      DCHECK(in.IsRegister());
2078      DCHECK(in.Equals(out));
2079      __ negl(out.AsRegister<Register>());
2080      break;
2081
2082    case Primitive::kPrimLong:
2083      DCHECK(in.IsRegisterPair());
2084      DCHECK(in.Equals(out));
2085      __ negl(out.AsRegisterPairLow<Register>());
2086      // Negation is similar to subtraction from zero.  The least
2087      // significant byte triggers a borrow when it is different from
2088      // zero; to take it into account, add 1 to the most significant
2089      // byte if the carry flag (CF) is set to 1 after the first NEGL
2090      // operation.
2091      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
2092      __ negl(out.AsRegisterPairHigh<Register>());
2093      break;
2094
2095    case Primitive::kPrimFloat: {
2096      DCHECK(in.Equals(out));
2097      Register constant = locations->GetTemp(0).AsRegister<Register>();
2098      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
2099      // Implement float negation with an exclusive or with value
2100      // 0x80000000 (mask for bit 31, representing the sign of a
2101      // single-precision floating-point number).
2102      __ movl(constant, Immediate(INT32_C(0x80000000)));
2103      __ movd(mask, constant);
2104      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
2105      break;
2106    }
2107
2108    case Primitive::kPrimDouble: {
2109      DCHECK(in.Equals(out));
2110      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2111      // Implement double negation with an exclusive or with value
2112      // 0x8000000000000000 (mask for bit 63, representing the sign of
2113      // a double-precision floating-point number).
2114      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
2115      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
2116      break;
2117    }
2118
2119    default:
2120      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
2121  }
2122}
2123
2124void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
2125  Primitive::Type result_type = conversion->GetResultType();
2126  Primitive::Type input_type = conversion->GetInputType();
2127  DCHECK_NE(result_type, input_type);
2128
2129  // The float-to-long and double-to-long type conversions rely on a
2130  // call to the runtime.
2131  LocationSummary::CallKind call_kind =
2132      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
2133       && result_type == Primitive::kPrimLong)
2134      ? LocationSummary::kCall
2135      : LocationSummary::kNoCall;
2136  LocationSummary* locations =
2137      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
2138
2139  // The Java language does not allow treating boolean as an integral type but
2140  // our bit representation makes it safe.
2141
2142  switch (result_type) {
2143    case Primitive::kPrimByte:
2144      switch (input_type) {
2145        case Primitive::kPrimBoolean:
2146          // Boolean input is a result of code transformations.
2147        case Primitive::kPrimShort:
2148        case Primitive::kPrimInt:
2149        case Primitive::kPrimChar:
2150          // Processing a Dex `int-to-byte' instruction.
2151          locations->SetInAt(0, Location::ByteRegisterOrConstant(ECX, conversion->InputAt(0)));
2152          // Make the output overlap to please the register allocator. This greatly simplifies
2153          // the validation of the linear scan implementation
2154          locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
2155          break;
2156
2157        default:
2158          LOG(FATAL) << "Unexpected type conversion from " << input_type
2159                     << " to " << result_type;
2160      }
2161      break;
2162
2163    case Primitive::kPrimShort:
2164      switch (input_type) {
2165        case Primitive::kPrimBoolean:
2166          // Boolean input is a result of code transformations.
2167        case Primitive::kPrimByte:
2168        case Primitive::kPrimInt:
2169        case Primitive::kPrimChar:
2170          // Processing a Dex `int-to-short' instruction.
2171          locations->SetInAt(0, Location::Any());
2172          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2173          break;
2174
2175        default:
2176          LOG(FATAL) << "Unexpected type conversion from " << input_type
2177                     << " to " << result_type;
2178      }
2179      break;
2180
2181    case Primitive::kPrimInt:
2182      switch (input_type) {
2183        case Primitive::kPrimLong:
2184          // Processing a Dex `long-to-int' instruction.
2185          locations->SetInAt(0, Location::Any());
2186          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2187          break;
2188
2189        case Primitive::kPrimFloat:
2190          // Processing a Dex `float-to-int' instruction.
2191          locations->SetInAt(0, Location::RequiresFpuRegister());
2192          locations->SetOut(Location::RequiresRegister());
2193          locations->AddTemp(Location::RequiresFpuRegister());
2194          break;
2195
2196        case Primitive::kPrimDouble:
2197          // Processing a Dex `double-to-int' instruction.
2198          locations->SetInAt(0, Location::RequiresFpuRegister());
2199          locations->SetOut(Location::RequiresRegister());
2200          locations->AddTemp(Location::RequiresFpuRegister());
2201          break;
2202
2203        default:
2204          LOG(FATAL) << "Unexpected type conversion from " << input_type
2205                     << " to " << result_type;
2206      }
2207      break;
2208
2209    case Primitive::kPrimLong:
2210      switch (input_type) {
2211        case Primitive::kPrimBoolean:
2212          // Boolean input is a result of code transformations.
2213        case Primitive::kPrimByte:
2214        case Primitive::kPrimShort:
2215        case Primitive::kPrimInt:
2216        case Primitive::kPrimChar:
2217          // Processing a Dex `int-to-long' instruction.
2218          locations->SetInAt(0, Location::RegisterLocation(EAX));
2219          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2220          break;
2221
2222        case Primitive::kPrimFloat:
2223        case Primitive::kPrimDouble: {
2224          // Processing a Dex `float-to-long' or 'double-to-long' instruction.
2225          InvokeRuntimeCallingConvention calling_convention;
2226          XmmRegister parameter = calling_convention.GetFpuRegisterAt(0);
2227          locations->SetInAt(0, Location::FpuRegisterLocation(parameter));
2228
2229          // The runtime helper puts the result in EAX, EDX.
2230          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2231        }
2232        break;
2233
2234        default:
2235          LOG(FATAL) << "Unexpected type conversion from " << input_type
2236                     << " to " << result_type;
2237      }
2238      break;
2239
2240    case Primitive::kPrimChar:
2241      switch (input_type) {
2242        case Primitive::kPrimBoolean:
2243          // Boolean input is a result of code transformations.
2244        case Primitive::kPrimByte:
2245        case Primitive::kPrimShort:
2246        case Primitive::kPrimInt:
2247          // Processing a Dex `int-to-char' instruction.
2248          locations->SetInAt(0, Location::Any());
2249          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2250          break;
2251
2252        default:
2253          LOG(FATAL) << "Unexpected type conversion from " << input_type
2254                     << " to " << result_type;
2255      }
2256      break;
2257
2258    case Primitive::kPrimFloat:
2259      switch (input_type) {
2260        case Primitive::kPrimBoolean:
2261          // Boolean input is a result of code transformations.
2262        case Primitive::kPrimByte:
2263        case Primitive::kPrimShort:
2264        case Primitive::kPrimInt:
2265        case Primitive::kPrimChar:
2266          // Processing a Dex `int-to-float' instruction.
2267          locations->SetInAt(0, Location::RequiresRegister());
2268          locations->SetOut(Location::RequiresFpuRegister());
2269          break;
2270
2271        case Primitive::kPrimLong:
2272          // Processing a Dex `long-to-float' instruction.
2273          locations->SetInAt(0, Location::Any());
2274          locations->SetOut(Location::Any());
2275          break;
2276
2277        case Primitive::kPrimDouble:
2278          // Processing a Dex `double-to-float' instruction.
2279          locations->SetInAt(0, Location::RequiresFpuRegister());
2280          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2281          break;
2282
2283        default:
2284          LOG(FATAL) << "Unexpected type conversion from " << input_type
2285                     << " to " << result_type;
2286      };
2287      break;
2288
2289    case Primitive::kPrimDouble:
2290      switch (input_type) {
2291        case Primitive::kPrimBoolean:
2292          // Boolean input is a result of code transformations.
2293        case Primitive::kPrimByte:
2294        case Primitive::kPrimShort:
2295        case Primitive::kPrimInt:
2296        case Primitive::kPrimChar:
2297          // Processing a Dex `int-to-double' instruction.
2298          locations->SetInAt(0, Location::RequiresRegister());
2299          locations->SetOut(Location::RequiresFpuRegister());
2300          break;
2301
2302        case Primitive::kPrimLong:
2303          // Processing a Dex `long-to-double' instruction.
2304          locations->SetInAt(0, Location::Any());
2305          locations->SetOut(Location::Any());
2306          break;
2307
2308        case Primitive::kPrimFloat:
2309          // Processing a Dex `float-to-double' instruction.
2310          locations->SetInAt(0, Location::RequiresFpuRegister());
2311          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2312          break;
2313
2314        default:
2315          LOG(FATAL) << "Unexpected type conversion from " << input_type
2316                     << " to " << result_type;
2317      }
2318      break;
2319
2320    default:
2321      LOG(FATAL) << "Unexpected type conversion from " << input_type
2322                 << " to " << result_type;
2323  }
2324}
2325
2326void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
2327  LocationSummary* locations = conversion->GetLocations();
2328  Location out = locations->Out();
2329  Location in = locations->InAt(0);
2330  Primitive::Type result_type = conversion->GetResultType();
2331  Primitive::Type input_type = conversion->GetInputType();
2332  DCHECK_NE(result_type, input_type);
2333  switch (result_type) {
2334    case Primitive::kPrimByte:
2335      switch (input_type) {
2336        case Primitive::kPrimBoolean:
2337          // Boolean input is a result of code transformations.
2338        case Primitive::kPrimShort:
2339        case Primitive::kPrimInt:
2340        case Primitive::kPrimChar:
2341          // Processing a Dex `int-to-byte' instruction.
2342          if (in.IsRegister()) {
2343            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
2344          } else {
2345            DCHECK(in.GetConstant()->IsIntConstant());
2346            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2347            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
2348          }
2349          break;
2350
2351        default:
2352          LOG(FATAL) << "Unexpected type conversion from " << input_type
2353                     << " to " << result_type;
2354      }
2355      break;
2356
2357    case Primitive::kPrimShort:
2358      switch (input_type) {
2359        case Primitive::kPrimBoolean:
2360          // Boolean input is a result of code transformations.
2361        case Primitive::kPrimByte:
2362        case Primitive::kPrimInt:
2363        case Primitive::kPrimChar:
2364          // Processing a Dex `int-to-short' instruction.
2365          if (in.IsRegister()) {
2366            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2367          } else if (in.IsStackSlot()) {
2368            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2369          } else {
2370            DCHECK(in.GetConstant()->IsIntConstant());
2371            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2372            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
2373          }
2374          break;
2375
2376        default:
2377          LOG(FATAL) << "Unexpected type conversion from " << input_type
2378                     << " to " << result_type;
2379      }
2380      break;
2381
2382    case Primitive::kPrimInt:
2383      switch (input_type) {
2384        case Primitive::kPrimLong:
2385          // Processing a Dex `long-to-int' instruction.
2386          if (in.IsRegisterPair()) {
2387            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
2388          } else if (in.IsDoubleStackSlot()) {
2389            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2390          } else {
2391            DCHECK(in.IsConstant());
2392            DCHECK(in.GetConstant()->IsLongConstant());
2393            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
2394            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
2395          }
2396          break;
2397
2398        case Primitive::kPrimFloat: {
2399          // Processing a Dex `float-to-int' instruction.
2400          XmmRegister input = in.AsFpuRegister<XmmRegister>();
2401          Register output = out.AsRegister<Register>();
2402          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2403          NearLabel done, nan;
2404
2405          __ movl(output, Immediate(kPrimIntMax));
2406          // temp = int-to-float(output)
2407          __ cvtsi2ss(temp, output);
2408          // if input >= temp goto done
2409          __ comiss(input, temp);
2410          __ j(kAboveEqual, &done);
2411          // if input == NaN goto nan
2412          __ j(kUnordered, &nan);
2413          // output = float-to-int-truncate(input)
2414          __ cvttss2si(output, input);
2415          __ jmp(&done);
2416          __ Bind(&nan);
2417          //  output = 0
2418          __ xorl(output, output);
2419          __ Bind(&done);
2420          break;
2421        }
2422
2423        case Primitive::kPrimDouble: {
2424          // Processing a Dex `double-to-int' instruction.
2425          XmmRegister input = in.AsFpuRegister<XmmRegister>();
2426          Register output = out.AsRegister<Register>();
2427          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2428          NearLabel done, nan;
2429
2430          __ movl(output, Immediate(kPrimIntMax));
2431          // temp = int-to-double(output)
2432          __ cvtsi2sd(temp, output);
2433          // if input >= temp goto done
2434          __ comisd(input, temp);
2435          __ j(kAboveEqual, &done);
2436          // if input == NaN goto nan
2437          __ j(kUnordered, &nan);
2438          // output = double-to-int-truncate(input)
2439          __ cvttsd2si(output, input);
2440          __ jmp(&done);
2441          __ Bind(&nan);
2442          //  output = 0
2443          __ xorl(output, output);
2444          __ Bind(&done);
2445          break;
2446        }
2447
2448        default:
2449          LOG(FATAL) << "Unexpected type conversion from " << input_type
2450                     << " to " << result_type;
2451      }
2452      break;
2453
2454    case Primitive::kPrimLong:
2455      switch (input_type) {
2456        case Primitive::kPrimBoolean:
2457          // Boolean input is a result of code transformations.
2458        case Primitive::kPrimByte:
2459        case Primitive::kPrimShort:
2460        case Primitive::kPrimInt:
2461        case Primitive::kPrimChar:
2462          // Processing a Dex `int-to-long' instruction.
2463          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
2464          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
2465          DCHECK_EQ(in.AsRegister<Register>(), EAX);
2466          __ cdq();
2467          break;
2468
2469        case Primitive::kPrimFloat:
2470          // Processing a Dex `float-to-long' instruction.
2471          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pF2l),
2472                                  conversion,
2473                                  conversion->GetDexPc(),
2474                                  nullptr);
2475          CheckEntrypointTypes<kQuickF2l, int64_t, float>();
2476          break;
2477
2478        case Primitive::kPrimDouble:
2479          // Processing a Dex `double-to-long' instruction.
2480          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pD2l),
2481                                  conversion,
2482                                  conversion->GetDexPc(),
2483                                  nullptr);
2484          CheckEntrypointTypes<kQuickD2l, int64_t, double>();
2485          break;
2486
2487        default:
2488          LOG(FATAL) << "Unexpected type conversion from " << input_type
2489                     << " to " << result_type;
2490      }
2491      break;
2492
2493    case Primitive::kPrimChar:
2494      switch (input_type) {
2495        case Primitive::kPrimBoolean:
2496          // Boolean input is a result of code transformations.
2497        case Primitive::kPrimByte:
2498        case Primitive::kPrimShort:
2499        case Primitive::kPrimInt:
2500          // Processing a Dex `Process a Dex `int-to-char'' instruction.
2501          if (in.IsRegister()) {
2502            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2503          } else if (in.IsStackSlot()) {
2504            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2505          } else {
2506            DCHECK(in.GetConstant()->IsIntConstant());
2507            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2508            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
2509          }
2510          break;
2511
2512        default:
2513          LOG(FATAL) << "Unexpected type conversion from " << input_type
2514                     << " to " << result_type;
2515      }
2516      break;
2517
2518    case Primitive::kPrimFloat:
2519      switch (input_type) {
2520        case Primitive::kPrimBoolean:
2521          // Boolean input is a result of code transformations.
2522        case Primitive::kPrimByte:
2523        case Primitive::kPrimShort:
2524        case Primitive::kPrimInt:
2525        case Primitive::kPrimChar:
2526          // Processing a Dex `int-to-float' instruction.
2527          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2528          break;
2529
2530        case Primitive::kPrimLong: {
2531          // Processing a Dex `long-to-float' instruction.
2532          size_t adjustment = 0;
2533
2534          // Create stack space for the call to
2535          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstps below.
2536          // TODO: enhance register allocator to ask for stack temporaries.
2537          if (!in.IsDoubleStackSlot() || !out.IsStackSlot()) {
2538            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2539            __ subl(ESP, Immediate(adjustment));
2540          }
2541
2542          // Load the value to the FP stack, using temporaries if needed.
2543          PushOntoFPStack(in, 0, adjustment, false, true);
2544
2545          if (out.IsStackSlot()) {
2546            __ fstps(Address(ESP, out.GetStackIndex() + adjustment));
2547          } else {
2548            __ fstps(Address(ESP, 0));
2549            Location stack_temp = Location::StackSlot(0);
2550            codegen_->Move32(out, stack_temp);
2551          }
2552
2553          // Remove the temporary stack space we allocated.
2554          if (adjustment != 0) {
2555            __ addl(ESP, Immediate(adjustment));
2556          }
2557          break;
2558        }
2559
2560        case Primitive::kPrimDouble:
2561          // Processing a Dex `double-to-float' instruction.
2562          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2563          break;
2564
2565        default:
2566          LOG(FATAL) << "Unexpected type conversion from " << input_type
2567                     << " to " << result_type;
2568      };
2569      break;
2570
2571    case Primitive::kPrimDouble:
2572      switch (input_type) {
2573        case Primitive::kPrimBoolean:
2574          // Boolean input is a result of code transformations.
2575        case Primitive::kPrimByte:
2576        case Primitive::kPrimShort:
2577        case Primitive::kPrimInt:
2578        case Primitive::kPrimChar:
2579          // Processing a Dex `int-to-double' instruction.
2580          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2581          break;
2582
2583        case Primitive::kPrimLong: {
2584          // Processing a Dex `long-to-double' instruction.
2585          size_t adjustment = 0;
2586
2587          // Create stack space for the call to
2588          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstpl below.
2589          // TODO: enhance register allocator to ask for stack temporaries.
2590          if (!in.IsDoubleStackSlot() || !out.IsDoubleStackSlot()) {
2591            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2592            __ subl(ESP, Immediate(adjustment));
2593          }
2594
2595          // Load the value to the FP stack, using temporaries if needed.
2596          PushOntoFPStack(in, 0, adjustment, false, true);
2597
2598          if (out.IsDoubleStackSlot()) {
2599            __ fstpl(Address(ESP, out.GetStackIndex() + adjustment));
2600          } else {
2601            __ fstpl(Address(ESP, 0));
2602            Location stack_temp = Location::DoubleStackSlot(0);
2603            codegen_->Move64(out, stack_temp);
2604          }
2605
2606          // Remove the temporary stack space we allocated.
2607          if (adjustment != 0) {
2608            __ addl(ESP, Immediate(adjustment));
2609          }
2610          break;
2611        }
2612
2613        case Primitive::kPrimFloat:
2614          // Processing a Dex `float-to-double' instruction.
2615          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2616          break;
2617
2618        default:
2619          LOG(FATAL) << "Unexpected type conversion from " << input_type
2620                     << " to " << result_type;
2621      };
2622      break;
2623
2624    default:
2625      LOG(FATAL) << "Unexpected type conversion from " << input_type
2626                 << " to " << result_type;
2627  }
2628}
2629
2630void LocationsBuilderX86::VisitAdd(HAdd* add) {
2631  LocationSummary* locations =
2632      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
2633  switch (add->GetResultType()) {
2634    case Primitive::kPrimInt: {
2635      locations->SetInAt(0, Location::RequiresRegister());
2636      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
2637      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2638      break;
2639    }
2640
2641    case Primitive::kPrimLong: {
2642      locations->SetInAt(0, Location::RequiresRegister());
2643      locations->SetInAt(1, Location::Any());
2644      locations->SetOut(Location::SameAsFirstInput());
2645      break;
2646    }
2647
2648    case Primitive::kPrimFloat:
2649    case Primitive::kPrimDouble: {
2650      locations->SetInAt(0, Location::RequiresFpuRegister());
2651      locations->SetInAt(1, Location::Any());
2652      locations->SetOut(Location::SameAsFirstInput());
2653      break;
2654    }
2655
2656    default:
2657      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2658      break;
2659  }
2660}
2661
2662void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
2663  LocationSummary* locations = add->GetLocations();
2664  Location first = locations->InAt(0);
2665  Location second = locations->InAt(1);
2666  Location out = locations->Out();
2667
2668  switch (add->GetResultType()) {
2669    case Primitive::kPrimInt: {
2670      if (second.IsRegister()) {
2671        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2672          __ addl(out.AsRegister<Register>(), second.AsRegister<Register>());
2673        } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
2674          __ addl(out.AsRegister<Register>(), first.AsRegister<Register>());
2675        } else {
2676          __ leal(out.AsRegister<Register>(), Address(
2677              first.AsRegister<Register>(), second.AsRegister<Register>(), TIMES_1, 0));
2678          }
2679      } else if (second.IsConstant()) {
2680        int32_t value = second.GetConstant()->AsIntConstant()->GetValue();
2681        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2682          __ addl(out.AsRegister<Register>(), Immediate(value));
2683        } else {
2684          __ leal(out.AsRegister<Register>(), Address(first.AsRegister<Register>(), value));
2685        }
2686      } else {
2687        DCHECK(first.Equals(locations->Out()));
2688        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2689      }
2690      break;
2691    }
2692
2693    case Primitive::kPrimLong: {
2694      if (second.IsRegisterPair()) {
2695        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2696        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2697      } else if (second.IsDoubleStackSlot()) {
2698        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2699        __ adcl(first.AsRegisterPairHigh<Register>(),
2700                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2701      } else {
2702        DCHECK(second.IsConstant()) << second;
2703        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2704        __ addl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2705        __ adcl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2706      }
2707      break;
2708    }
2709
2710    case Primitive::kPrimFloat: {
2711      if (second.IsFpuRegister()) {
2712        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2713      } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2714        HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2715        DCHECK(!const_area->NeedsMaterialization());
2716        __ addss(first.AsFpuRegister<XmmRegister>(),
2717                 codegen_->LiteralFloatAddress(
2718                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2719                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2720      } else {
2721        DCHECK(second.IsStackSlot());
2722        __ addss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2723      }
2724      break;
2725    }
2726
2727    case Primitive::kPrimDouble: {
2728      if (second.IsFpuRegister()) {
2729        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2730      } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2731        HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2732        DCHECK(!const_area->NeedsMaterialization());
2733        __ addsd(first.AsFpuRegister<XmmRegister>(),
2734                 codegen_->LiteralDoubleAddress(
2735                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2736                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2737      } else {
2738        DCHECK(second.IsDoubleStackSlot());
2739        __ addsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2740      }
2741      break;
2742    }
2743
2744    default:
2745      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2746  }
2747}
2748
2749void LocationsBuilderX86::VisitSub(HSub* sub) {
2750  LocationSummary* locations =
2751      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2752  switch (sub->GetResultType()) {
2753    case Primitive::kPrimInt:
2754    case Primitive::kPrimLong: {
2755      locations->SetInAt(0, Location::RequiresRegister());
2756      locations->SetInAt(1, Location::Any());
2757      locations->SetOut(Location::SameAsFirstInput());
2758      break;
2759    }
2760    case Primitive::kPrimFloat:
2761    case Primitive::kPrimDouble: {
2762      locations->SetInAt(0, Location::RequiresFpuRegister());
2763      locations->SetInAt(1, Location::Any());
2764      locations->SetOut(Location::SameAsFirstInput());
2765      break;
2766    }
2767
2768    default:
2769      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2770  }
2771}
2772
2773void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
2774  LocationSummary* locations = sub->GetLocations();
2775  Location first = locations->InAt(0);
2776  Location second = locations->InAt(1);
2777  DCHECK(first.Equals(locations->Out()));
2778  switch (sub->GetResultType()) {
2779    case Primitive::kPrimInt: {
2780      if (second.IsRegister()) {
2781        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
2782      } else if (second.IsConstant()) {
2783        __ subl(first.AsRegister<Register>(),
2784                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2785      } else {
2786        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2787      }
2788      break;
2789    }
2790
2791    case Primitive::kPrimLong: {
2792      if (second.IsRegisterPair()) {
2793        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2794        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2795      } else if (second.IsDoubleStackSlot()) {
2796        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2797        __ sbbl(first.AsRegisterPairHigh<Register>(),
2798                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2799      } else {
2800        DCHECK(second.IsConstant()) << second;
2801        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2802        __ subl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2803        __ sbbl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2804      }
2805      break;
2806    }
2807
2808    case Primitive::kPrimFloat: {
2809      if (second.IsFpuRegister()) {
2810        __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2811      } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2812        HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2813        DCHECK(!const_area->NeedsMaterialization());
2814        __ subss(first.AsFpuRegister<XmmRegister>(),
2815                 codegen_->LiteralFloatAddress(
2816                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2817                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2818      } else {
2819        DCHECK(second.IsStackSlot());
2820        __ subss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2821      }
2822      break;
2823    }
2824
2825    case Primitive::kPrimDouble: {
2826      if (second.IsFpuRegister()) {
2827        __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2828      } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2829        HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2830        DCHECK(!const_area->NeedsMaterialization());
2831        __ subsd(first.AsFpuRegister<XmmRegister>(),
2832                 codegen_->LiteralDoubleAddress(
2833                     const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2834                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2835      } else {
2836        DCHECK(second.IsDoubleStackSlot());
2837        __ subsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2838      }
2839      break;
2840    }
2841
2842    default:
2843      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2844  }
2845}
2846
2847void LocationsBuilderX86::VisitMul(HMul* mul) {
2848  LocationSummary* locations =
2849      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2850  switch (mul->GetResultType()) {
2851    case Primitive::kPrimInt:
2852      locations->SetInAt(0, Location::RequiresRegister());
2853      locations->SetInAt(1, Location::Any());
2854      if (mul->InputAt(1)->IsIntConstant()) {
2855        // Can use 3 operand multiply.
2856        locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2857      } else {
2858        locations->SetOut(Location::SameAsFirstInput());
2859      }
2860      break;
2861    case Primitive::kPrimLong: {
2862      locations->SetInAt(0, Location::RequiresRegister());
2863      locations->SetInAt(1, Location::Any());
2864      locations->SetOut(Location::SameAsFirstInput());
2865      // Needed for imul on 32bits with 64bits output.
2866      locations->AddTemp(Location::RegisterLocation(EAX));
2867      locations->AddTemp(Location::RegisterLocation(EDX));
2868      break;
2869    }
2870    case Primitive::kPrimFloat:
2871    case Primitive::kPrimDouble: {
2872      locations->SetInAt(0, Location::RequiresFpuRegister());
2873      locations->SetInAt(1, Location::Any());
2874      locations->SetOut(Location::SameAsFirstInput());
2875      break;
2876    }
2877
2878    default:
2879      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2880  }
2881}
2882
2883void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2884  LocationSummary* locations = mul->GetLocations();
2885  Location first = locations->InAt(0);
2886  Location second = locations->InAt(1);
2887  Location out = locations->Out();
2888
2889  switch (mul->GetResultType()) {
2890    case Primitive::kPrimInt:
2891      // The constant may have ended up in a register, so test explicitly to avoid
2892      // problems where the output may not be the same as the first operand.
2893      if (mul->InputAt(1)->IsIntConstant()) {
2894        Immediate imm(mul->InputAt(1)->AsIntConstant()->GetValue());
2895        __ imull(out.AsRegister<Register>(), first.AsRegister<Register>(), imm);
2896      } else if (second.IsRegister()) {
2897        DCHECK(first.Equals(out));
2898        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
2899      } else {
2900        DCHECK(second.IsStackSlot());
2901        DCHECK(first.Equals(out));
2902        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2903      }
2904      break;
2905
2906    case Primitive::kPrimLong: {
2907      Register in1_hi = first.AsRegisterPairHigh<Register>();
2908      Register in1_lo = first.AsRegisterPairLow<Register>();
2909      Register eax = locations->GetTemp(0).AsRegister<Register>();
2910      Register edx = locations->GetTemp(1).AsRegister<Register>();
2911
2912      DCHECK_EQ(EAX, eax);
2913      DCHECK_EQ(EDX, edx);
2914
2915      // input: in1 - 64 bits, in2 - 64 bits.
2916      // output: in1
2917      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2918      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2919      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
2920      if (second.IsConstant()) {
2921        DCHECK(second.GetConstant()->IsLongConstant());
2922
2923        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2924        int32_t low_value = Low32Bits(value);
2925        int32_t high_value = High32Bits(value);
2926        Immediate low(low_value);
2927        Immediate high(high_value);
2928
2929        __ movl(eax, high);
2930        // eax <- in1.lo * in2.hi
2931        __ imull(eax, in1_lo);
2932        // in1.hi <- in1.hi * in2.lo
2933        __ imull(in1_hi, low);
2934        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2935        __ addl(in1_hi, eax);
2936        // move in2_lo to eax to prepare for double precision
2937        __ movl(eax, low);
2938        // edx:eax <- in1.lo * in2.lo
2939        __ mull(in1_lo);
2940        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2941        __ addl(in1_hi, edx);
2942        // in1.lo <- (in1.lo * in2.lo)[31:0];
2943        __ movl(in1_lo, eax);
2944      } else if (second.IsRegisterPair()) {
2945        Register in2_hi = second.AsRegisterPairHigh<Register>();
2946        Register in2_lo = second.AsRegisterPairLow<Register>();
2947
2948        __ movl(eax, in2_hi);
2949        // eax <- in1.lo * in2.hi
2950        __ imull(eax, in1_lo);
2951        // in1.hi <- in1.hi * in2.lo
2952        __ imull(in1_hi, in2_lo);
2953        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2954        __ addl(in1_hi, eax);
2955        // move in1_lo to eax to prepare for double precision
2956        __ movl(eax, in1_lo);
2957        // edx:eax <- in1.lo * in2.lo
2958        __ mull(in2_lo);
2959        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2960        __ addl(in1_hi, edx);
2961        // in1.lo <- (in1.lo * in2.lo)[31:0];
2962        __ movl(in1_lo, eax);
2963      } else {
2964        DCHECK(second.IsDoubleStackSlot()) << second;
2965        Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
2966        Address in2_lo(ESP, second.GetStackIndex());
2967
2968        __ movl(eax, in2_hi);
2969        // eax <- in1.lo * in2.hi
2970        __ imull(eax, in1_lo);
2971        // in1.hi <- in1.hi * in2.lo
2972        __ imull(in1_hi, in2_lo);
2973        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2974        __ addl(in1_hi, eax);
2975        // move in1_lo to eax to prepare for double precision
2976        __ movl(eax, in1_lo);
2977        // edx:eax <- in1.lo * in2.lo
2978        __ mull(in2_lo);
2979        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2980        __ addl(in1_hi, edx);
2981        // in1.lo <- (in1.lo * in2.lo)[31:0];
2982        __ movl(in1_lo, eax);
2983      }
2984
2985      break;
2986    }
2987
2988    case Primitive::kPrimFloat: {
2989      DCHECK(first.Equals(locations->Out()));
2990      if (second.IsFpuRegister()) {
2991        __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2992      } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
2993        HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
2994        DCHECK(!const_area->NeedsMaterialization());
2995        __ mulss(first.AsFpuRegister<XmmRegister>(),
2996                 codegen_->LiteralFloatAddress(
2997                     const_area->GetConstant()->AsFloatConstant()->GetValue(),
2998                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2999      } else {
3000        DCHECK(second.IsStackSlot());
3001        __ mulss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3002      }
3003      break;
3004    }
3005
3006    case Primitive::kPrimDouble: {
3007      DCHECK(first.Equals(locations->Out()));
3008      if (second.IsFpuRegister()) {
3009        __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3010      } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
3011        HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
3012        DCHECK(!const_area->NeedsMaterialization());
3013        __ mulsd(first.AsFpuRegister<XmmRegister>(),
3014                 codegen_->LiteralDoubleAddress(
3015                     const_area->GetConstant()->AsDoubleConstant()->GetValue(),
3016                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3017      } else {
3018        DCHECK(second.IsDoubleStackSlot());
3019        __ mulsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3020      }
3021      break;
3022    }
3023
3024    default:
3025      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
3026  }
3027}
3028
3029void InstructionCodeGeneratorX86::PushOntoFPStack(Location source,
3030                                                  uint32_t temp_offset,
3031                                                  uint32_t stack_adjustment,
3032                                                  bool is_fp,
3033                                                  bool is_wide) {
3034  if (source.IsStackSlot()) {
3035    DCHECK(!is_wide);
3036    if (is_fp) {
3037      __ flds(Address(ESP, source.GetStackIndex() + stack_adjustment));
3038    } else {
3039      __ filds(Address(ESP, source.GetStackIndex() + stack_adjustment));
3040    }
3041  } else if (source.IsDoubleStackSlot()) {
3042    DCHECK(is_wide);
3043    if (is_fp) {
3044      __ fldl(Address(ESP, source.GetStackIndex() + stack_adjustment));
3045    } else {
3046      __ fildl(Address(ESP, source.GetStackIndex() + stack_adjustment));
3047    }
3048  } else {
3049    // Write the value to the temporary location on the stack and load to FP stack.
3050    if (!is_wide) {
3051      Location stack_temp = Location::StackSlot(temp_offset);
3052      codegen_->Move32(stack_temp, source);
3053      if (is_fp) {
3054        __ flds(Address(ESP, temp_offset));
3055      } else {
3056        __ filds(Address(ESP, temp_offset));
3057      }
3058    } else {
3059      Location stack_temp = Location::DoubleStackSlot(temp_offset);
3060      codegen_->Move64(stack_temp, source);
3061      if (is_fp) {
3062        __ fldl(Address(ESP, temp_offset));
3063      } else {
3064        __ fildl(Address(ESP, temp_offset));
3065      }
3066    }
3067  }
3068}
3069
3070void InstructionCodeGeneratorX86::GenerateRemFP(HRem *rem) {
3071  Primitive::Type type = rem->GetResultType();
3072  bool is_float = type == Primitive::kPrimFloat;
3073  size_t elem_size = Primitive::ComponentSize(type);
3074  LocationSummary* locations = rem->GetLocations();
3075  Location first = locations->InAt(0);
3076  Location second = locations->InAt(1);
3077  Location out = locations->Out();
3078
3079  // Create stack space for 2 elements.
3080  // TODO: enhance register allocator to ask for stack temporaries.
3081  __ subl(ESP, Immediate(2 * elem_size));
3082
3083  // Load the values to the FP stack in reverse order, using temporaries if needed.
3084  const bool is_wide = !is_float;
3085  PushOntoFPStack(second, elem_size, 2 * elem_size, /* is_fp */ true, is_wide);
3086  PushOntoFPStack(first, 0, 2 * elem_size, /* is_fp */ true, is_wide);
3087
3088  // Loop doing FPREM until we stabilize.
3089  NearLabel retry;
3090  __ Bind(&retry);
3091  __ fprem();
3092
3093  // Move FP status to AX.
3094  __ fstsw();
3095
3096  // And see if the argument reduction is complete. This is signaled by the
3097  // C2 FPU flag bit set to 0.
3098  __ andl(EAX, Immediate(kC2ConditionMask));
3099  __ j(kNotEqual, &retry);
3100
3101  // We have settled on the final value. Retrieve it into an XMM register.
3102  // Store FP top of stack to real stack.
3103  if (is_float) {
3104    __ fsts(Address(ESP, 0));
3105  } else {
3106    __ fstl(Address(ESP, 0));
3107  }
3108
3109  // Pop the 2 items from the FP stack.
3110  __ fucompp();
3111
3112  // Load the value from the stack into an XMM register.
3113  DCHECK(out.IsFpuRegister()) << out;
3114  if (is_float) {
3115    __ movss(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
3116  } else {
3117    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
3118  }
3119
3120  // And remove the temporary stack space we allocated.
3121  __ addl(ESP, Immediate(2 * elem_size));
3122}
3123
3124
3125void InstructionCodeGeneratorX86::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
3126  DCHECK(instruction->IsDiv() || instruction->IsRem());
3127
3128  LocationSummary* locations = instruction->GetLocations();
3129  DCHECK(locations->InAt(1).IsConstant());
3130  DCHECK(locations->InAt(1).GetConstant()->IsIntConstant());
3131
3132  Register out_register = locations->Out().AsRegister<Register>();
3133  Register input_register = locations->InAt(0).AsRegister<Register>();
3134  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3135
3136  DCHECK(imm == 1 || imm == -1);
3137
3138  if (instruction->IsRem()) {
3139    __ xorl(out_register, out_register);
3140  } else {
3141    __ movl(out_register, input_register);
3142    if (imm == -1) {
3143      __ negl(out_register);
3144    }
3145  }
3146}
3147
3148
3149void InstructionCodeGeneratorX86::DivByPowerOfTwo(HDiv* instruction) {
3150  LocationSummary* locations = instruction->GetLocations();
3151
3152  Register out_register = locations->Out().AsRegister<Register>();
3153  Register input_register = locations->InAt(0).AsRegister<Register>();
3154  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3155
3156  DCHECK(IsPowerOfTwo(std::abs(imm)));
3157  Register num = locations->GetTemp(0).AsRegister<Register>();
3158
3159  __ leal(num, Address(input_register, std::abs(imm) - 1));
3160  __ testl(input_register, input_register);
3161  __ cmovl(kGreaterEqual, num, input_register);
3162  int shift = CTZ(imm);
3163  __ sarl(num, Immediate(shift));
3164
3165  if (imm < 0) {
3166    __ negl(num);
3167  }
3168
3169  __ movl(out_register, num);
3170}
3171
3172void InstructionCodeGeneratorX86::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
3173  DCHECK(instruction->IsDiv() || instruction->IsRem());
3174
3175  LocationSummary* locations = instruction->GetLocations();
3176  int imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3177
3178  Register eax = locations->InAt(0).AsRegister<Register>();
3179  Register out = locations->Out().AsRegister<Register>();
3180  Register num;
3181  Register edx;
3182
3183  if (instruction->IsDiv()) {
3184    edx = locations->GetTemp(0).AsRegister<Register>();
3185    num = locations->GetTemp(1).AsRegister<Register>();
3186  } else {
3187    edx = locations->Out().AsRegister<Register>();
3188    num = locations->GetTemp(0).AsRegister<Register>();
3189  }
3190
3191  DCHECK_EQ(EAX, eax);
3192  DCHECK_EQ(EDX, edx);
3193  if (instruction->IsDiv()) {
3194    DCHECK_EQ(EAX, out);
3195  } else {
3196    DCHECK_EQ(EDX, out);
3197  }
3198
3199  int64_t magic;
3200  int shift;
3201  CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
3202
3203  NearLabel ndiv;
3204  NearLabel end;
3205  // If numerator is 0, the result is 0, no computation needed.
3206  __ testl(eax, eax);
3207  __ j(kNotEqual, &ndiv);
3208
3209  __ xorl(out, out);
3210  __ jmp(&end);
3211
3212  __ Bind(&ndiv);
3213
3214  // Save the numerator.
3215  __ movl(num, eax);
3216
3217  // EAX = magic
3218  __ movl(eax, Immediate(magic));
3219
3220  // EDX:EAX = magic * numerator
3221  __ imull(num);
3222
3223  if (imm > 0 && magic < 0) {
3224    // EDX += num
3225    __ addl(edx, num);
3226  } else if (imm < 0 && magic > 0) {
3227    __ subl(edx, num);
3228  }
3229
3230  // Shift if needed.
3231  if (shift != 0) {
3232    __ sarl(edx, Immediate(shift));
3233  }
3234
3235  // EDX += 1 if EDX < 0
3236  __ movl(eax, edx);
3237  __ shrl(edx, Immediate(31));
3238  __ addl(edx, eax);
3239
3240  if (instruction->IsRem()) {
3241    __ movl(eax, num);
3242    __ imull(edx, Immediate(imm));
3243    __ subl(eax, edx);
3244    __ movl(edx, eax);
3245  } else {
3246    __ movl(eax, edx);
3247  }
3248  __ Bind(&end);
3249}
3250
3251void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
3252  DCHECK(instruction->IsDiv() || instruction->IsRem());
3253
3254  LocationSummary* locations = instruction->GetLocations();
3255  Location out = locations->Out();
3256  Location first = locations->InAt(0);
3257  Location second = locations->InAt(1);
3258  bool is_div = instruction->IsDiv();
3259
3260  switch (instruction->GetResultType()) {
3261    case Primitive::kPrimInt: {
3262      DCHECK_EQ(EAX, first.AsRegister<Register>());
3263      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
3264
3265      if (second.IsConstant()) {
3266        int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
3267
3268        if (imm == 0) {
3269          // Do not generate anything for 0. DivZeroCheck would forbid any generated code.
3270        } else if (imm == 1 || imm == -1) {
3271          DivRemOneOrMinusOne(instruction);
3272        } else if (is_div && IsPowerOfTwo(std::abs(imm))) {
3273          DivByPowerOfTwo(instruction->AsDiv());
3274        } else {
3275          DCHECK(imm <= -2 || imm >= 2);
3276          GenerateDivRemWithAnyConstant(instruction);
3277        }
3278      } else {
3279        SlowPathCode* slow_path =
3280          new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(out.AsRegister<Register>(),
3281              is_div);
3282        codegen_->AddSlowPath(slow_path);
3283
3284        Register second_reg = second.AsRegister<Register>();
3285        // 0x80000000/-1 triggers an arithmetic exception!
3286        // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
3287        // it's safe to just use negl instead of more complex comparisons.
3288
3289        __ cmpl(second_reg, Immediate(-1));
3290        __ j(kEqual, slow_path->GetEntryLabel());
3291
3292        // edx:eax <- sign-extended of eax
3293        __ cdq();
3294        // eax = quotient, edx = remainder
3295        __ idivl(second_reg);
3296        __ Bind(slow_path->GetExitLabel());
3297      }
3298      break;
3299    }
3300
3301    case Primitive::kPrimLong: {
3302      InvokeRuntimeCallingConvention calling_convention;
3303      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
3304      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
3305      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
3306      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
3307      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
3308      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
3309
3310      if (is_div) {
3311        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLdiv),
3312                                instruction,
3313                                instruction->GetDexPc(),
3314                                nullptr);
3315        CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>();
3316      } else {
3317        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLmod),
3318                                instruction,
3319                                instruction->GetDexPc(),
3320                                nullptr);
3321        CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>();
3322      }
3323      break;
3324    }
3325
3326    default:
3327      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
3328  }
3329}
3330
3331void LocationsBuilderX86::VisitDiv(HDiv* div) {
3332  LocationSummary::CallKind call_kind = (div->GetResultType() == Primitive::kPrimLong)
3333      ? LocationSummary::kCall
3334      : LocationSummary::kNoCall;
3335  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
3336
3337  switch (div->GetResultType()) {
3338    case Primitive::kPrimInt: {
3339      locations->SetInAt(0, Location::RegisterLocation(EAX));
3340      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
3341      locations->SetOut(Location::SameAsFirstInput());
3342      // Intel uses edx:eax as the dividend.
3343      locations->AddTemp(Location::RegisterLocation(EDX));
3344      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
3345      // which enforces results to be in EAX and EDX, things are simpler if we use EAX also as
3346      // output and request another temp.
3347      if (div->InputAt(1)->IsIntConstant()) {
3348        locations->AddTemp(Location::RequiresRegister());
3349      }
3350      break;
3351    }
3352    case Primitive::kPrimLong: {
3353      InvokeRuntimeCallingConvention calling_convention;
3354      locations->SetInAt(0, Location::RegisterPairLocation(
3355          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
3356      locations->SetInAt(1, Location::RegisterPairLocation(
3357          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
3358      // Runtime helper puts the result in EAX, EDX.
3359      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
3360      break;
3361    }
3362    case Primitive::kPrimFloat:
3363    case Primitive::kPrimDouble: {
3364      locations->SetInAt(0, Location::RequiresFpuRegister());
3365      locations->SetInAt(1, Location::Any());
3366      locations->SetOut(Location::SameAsFirstInput());
3367      break;
3368    }
3369
3370    default:
3371      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
3372  }
3373}
3374
3375void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
3376  LocationSummary* locations = div->GetLocations();
3377  Location first = locations->InAt(0);
3378  Location second = locations->InAt(1);
3379
3380  switch (div->GetResultType()) {
3381    case Primitive::kPrimInt:
3382    case Primitive::kPrimLong: {
3383      GenerateDivRemIntegral(div);
3384      break;
3385    }
3386
3387    case Primitive::kPrimFloat: {
3388      if (second.IsFpuRegister()) {
3389        __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3390      } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3391        HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3392        DCHECK(!const_area->NeedsMaterialization());
3393        __ divss(first.AsFpuRegister<XmmRegister>(),
3394                 codegen_->LiteralFloatAddress(
3395                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
3396                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3397      } else {
3398        DCHECK(second.IsStackSlot());
3399        __ divss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3400      }
3401      break;
3402    }
3403
3404    case Primitive::kPrimDouble: {
3405      if (second.IsFpuRegister()) {
3406        __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3407      } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3408        HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3409        DCHECK(!const_area->NeedsMaterialization());
3410        __ divsd(first.AsFpuRegister<XmmRegister>(),
3411                 codegen_->LiteralDoubleAddress(
3412                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
3413                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3414      } else {
3415        DCHECK(second.IsDoubleStackSlot());
3416        __ divsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3417      }
3418      break;
3419    }
3420
3421    default:
3422      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
3423  }
3424}
3425
3426void LocationsBuilderX86::VisitRem(HRem* rem) {
3427  Primitive::Type type = rem->GetResultType();
3428
3429  LocationSummary::CallKind call_kind = (rem->GetResultType() == Primitive::kPrimLong)
3430      ? LocationSummary::kCall
3431      : LocationSummary::kNoCall;
3432  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
3433
3434  switch (type) {
3435    case Primitive::kPrimInt: {
3436      locations->SetInAt(0, Location::RegisterLocation(EAX));
3437      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
3438      locations->SetOut(Location::RegisterLocation(EDX));
3439      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
3440      // which enforces results to be in EAX and EDX, things are simpler if we use EDX also as
3441      // output and request another temp.
3442      if (rem->InputAt(1)->IsIntConstant()) {
3443        locations->AddTemp(Location::RequiresRegister());
3444      }
3445      break;
3446    }
3447    case Primitive::kPrimLong: {
3448      InvokeRuntimeCallingConvention calling_convention;
3449      locations->SetInAt(0, Location::RegisterPairLocation(
3450          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
3451      locations->SetInAt(1, Location::RegisterPairLocation(
3452          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
3453      // Runtime helper puts the result in EAX, EDX.
3454      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
3455      break;
3456    }
3457    case Primitive::kPrimDouble:
3458    case Primitive::kPrimFloat: {
3459      locations->SetInAt(0, Location::Any());
3460      locations->SetInAt(1, Location::Any());
3461      locations->SetOut(Location::RequiresFpuRegister());
3462      locations->AddTemp(Location::RegisterLocation(EAX));
3463      break;
3464    }
3465
3466    default:
3467      LOG(FATAL) << "Unexpected rem type " << type;
3468  }
3469}
3470
3471void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
3472  Primitive::Type type = rem->GetResultType();
3473  switch (type) {
3474    case Primitive::kPrimInt:
3475    case Primitive::kPrimLong: {
3476      GenerateDivRemIntegral(rem);
3477      break;
3478    }
3479    case Primitive::kPrimFloat:
3480    case Primitive::kPrimDouble: {
3481      GenerateRemFP(rem);
3482      break;
3483    }
3484    default:
3485      LOG(FATAL) << "Unexpected rem type " << type;
3486  }
3487}
3488
3489void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3490  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
3491      ? LocationSummary::kCallOnSlowPath
3492      : LocationSummary::kNoCall;
3493  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3494  switch (instruction->GetType()) {
3495    case Primitive::kPrimByte:
3496    case Primitive::kPrimChar:
3497    case Primitive::kPrimShort:
3498    case Primitive::kPrimInt: {
3499      locations->SetInAt(0, Location::Any());
3500      break;
3501    }
3502    case Primitive::kPrimLong: {
3503      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3504      if (!instruction->IsConstant()) {
3505        locations->AddTemp(Location::RequiresRegister());
3506      }
3507      break;
3508    }
3509    default:
3510      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
3511  }
3512  if (instruction->HasUses()) {
3513    locations->SetOut(Location::SameAsFirstInput());
3514  }
3515}
3516
3517void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3518  SlowPathCode* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
3519  codegen_->AddSlowPath(slow_path);
3520
3521  LocationSummary* locations = instruction->GetLocations();
3522  Location value = locations->InAt(0);
3523
3524  switch (instruction->GetType()) {
3525    case Primitive::kPrimByte:
3526    case Primitive::kPrimChar:
3527    case Primitive::kPrimShort:
3528    case Primitive::kPrimInt: {
3529      if (value.IsRegister()) {
3530        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
3531        __ j(kEqual, slow_path->GetEntryLabel());
3532      } else if (value.IsStackSlot()) {
3533        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
3534        __ j(kEqual, slow_path->GetEntryLabel());
3535      } else {
3536        DCHECK(value.IsConstant()) << value;
3537        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
3538        __ jmp(slow_path->GetEntryLabel());
3539        }
3540      }
3541      break;
3542    }
3543    case Primitive::kPrimLong: {
3544      if (value.IsRegisterPair()) {
3545        Register temp = locations->GetTemp(0).AsRegister<Register>();
3546        __ movl(temp, value.AsRegisterPairLow<Register>());
3547        __ orl(temp, value.AsRegisterPairHigh<Register>());
3548        __ j(kEqual, slow_path->GetEntryLabel());
3549      } else {
3550        DCHECK(value.IsConstant()) << value;
3551        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
3552          __ jmp(slow_path->GetEntryLabel());
3553        }
3554      }
3555      break;
3556    }
3557    default:
3558      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
3559  }
3560}
3561
3562void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
3563  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3564
3565  LocationSummary* locations =
3566      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
3567
3568  switch (op->GetResultType()) {
3569    case Primitive::kPrimInt:
3570    case Primitive::kPrimLong: {
3571      // Can't have Location::Any() and output SameAsFirstInput()
3572      locations->SetInAt(0, Location::RequiresRegister());
3573      // The shift count needs to be in CL or a constant.
3574      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
3575      locations->SetOut(Location::SameAsFirstInput());
3576      break;
3577    }
3578    default:
3579      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3580  }
3581}
3582
3583void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
3584  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3585
3586  LocationSummary* locations = op->GetLocations();
3587  Location first = locations->InAt(0);
3588  Location second = locations->InAt(1);
3589  DCHECK(first.Equals(locations->Out()));
3590
3591  switch (op->GetResultType()) {
3592    case Primitive::kPrimInt: {
3593      DCHECK(first.IsRegister());
3594      Register first_reg = first.AsRegister<Register>();
3595      if (second.IsRegister()) {
3596        Register second_reg = second.AsRegister<Register>();
3597        DCHECK_EQ(ECX, second_reg);
3598        if (op->IsShl()) {
3599          __ shll(first_reg, second_reg);
3600        } else if (op->IsShr()) {
3601          __ sarl(first_reg, second_reg);
3602        } else {
3603          __ shrl(first_reg, second_reg);
3604        }
3605      } else {
3606        int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue;
3607        if (shift == 0) {
3608          return;
3609        }
3610        Immediate imm(shift);
3611        if (op->IsShl()) {
3612          __ shll(first_reg, imm);
3613        } else if (op->IsShr()) {
3614          __ sarl(first_reg, imm);
3615        } else {
3616          __ shrl(first_reg, imm);
3617        }
3618      }
3619      break;
3620    }
3621    case Primitive::kPrimLong: {
3622      if (second.IsRegister()) {
3623        Register second_reg = second.AsRegister<Register>();
3624        DCHECK_EQ(ECX, second_reg);
3625        if (op->IsShl()) {
3626          GenerateShlLong(first, second_reg);
3627        } else if (op->IsShr()) {
3628          GenerateShrLong(first, second_reg);
3629        } else {
3630          GenerateUShrLong(first, second_reg);
3631        }
3632      } else {
3633        // Shift by a constant.
3634        int shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue;
3635        // Nothing to do if the shift is 0, as the input is already the output.
3636        if (shift != 0) {
3637          if (op->IsShl()) {
3638            GenerateShlLong(first, shift);
3639          } else if (op->IsShr()) {
3640            GenerateShrLong(first, shift);
3641          } else {
3642            GenerateUShrLong(first, shift);
3643          }
3644        }
3645      }
3646      break;
3647    }
3648    default:
3649      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3650  }
3651}
3652
3653void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, int shift) {
3654  Register low = loc.AsRegisterPairLow<Register>();
3655  Register high = loc.AsRegisterPairHigh<Register>();
3656  if (shift == 1) {
3657    // This is just an addition.
3658    __ addl(low, low);
3659    __ adcl(high, high);
3660  } else if (shift == 32) {
3661    // Shift by 32 is easy. High gets low, and low gets 0.
3662    codegen_->EmitParallelMoves(
3663        loc.ToLow(),
3664        loc.ToHigh(),
3665        Primitive::kPrimInt,
3666        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3667        loc.ToLow(),
3668        Primitive::kPrimInt);
3669  } else if (shift > 32) {
3670    // Low part becomes 0.  High part is low part << (shift-32).
3671    __ movl(high, low);
3672    __ shll(high, Immediate(shift - 32));
3673    __ xorl(low, low);
3674  } else {
3675    // Between 1 and 31.
3676    __ shld(high, low, Immediate(shift));
3677    __ shll(low, Immediate(shift));
3678  }
3679}
3680
3681void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
3682  NearLabel done;
3683  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
3684  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
3685  __ testl(shifter, Immediate(32));
3686  __ j(kEqual, &done);
3687  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
3688  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
3689  __ Bind(&done);
3690}
3691
3692void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, int shift) {
3693  Register low = loc.AsRegisterPairLow<Register>();
3694  Register high = loc.AsRegisterPairHigh<Register>();
3695  if (shift == 32) {
3696    // Need to copy the sign.
3697    DCHECK_NE(low, high);
3698    __ movl(low, high);
3699    __ sarl(high, Immediate(31));
3700  } else if (shift > 32) {
3701    DCHECK_NE(low, high);
3702    // High part becomes sign. Low part is shifted by shift - 32.
3703    __ movl(low, high);
3704    __ sarl(high, Immediate(31));
3705    __ sarl(low, Immediate(shift - 32));
3706  } else {
3707    // Between 1 and 31.
3708    __ shrd(low, high, Immediate(shift));
3709    __ sarl(high, Immediate(shift));
3710  }
3711}
3712
3713void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
3714  NearLabel done;
3715  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3716  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
3717  __ testl(shifter, Immediate(32));
3718  __ j(kEqual, &done);
3719  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3720  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
3721  __ Bind(&done);
3722}
3723
3724void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, int shift) {
3725  Register low = loc.AsRegisterPairLow<Register>();
3726  Register high = loc.AsRegisterPairHigh<Register>();
3727  if (shift == 32) {
3728    // Shift by 32 is easy. Low gets high, and high gets 0.
3729    codegen_->EmitParallelMoves(
3730        loc.ToHigh(),
3731        loc.ToLow(),
3732        Primitive::kPrimInt,
3733        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3734        loc.ToHigh(),
3735        Primitive::kPrimInt);
3736  } else if (shift > 32) {
3737    // Low part is high >> (shift - 32). High part becomes 0.
3738    __ movl(low, high);
3739    __ shrl(low, Immediate(shift - 32));
3740    __ xorl(high, high);
3741  } else {
3742    // Between 1 and 31.
3743    __ shrd(low, high, Immediate(shift));
3744    __ shrl(high, Immediate(shift));
3745  }
3746}
3747
3748void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
3749  NearLabel done;
3750  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3751  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
3752  __ testl(shifter, Immediate(32));
3753  __ j(kEqual, &done);
3754  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3755  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
3756  __ Bind(&done);
3757}
3758
3759void LocationsBuilderX86::VisitShl(HShl* shl) {
3760  HandleShift(shl);
3761}
3762
3763void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
3764  HandleShift(shl);
3765}
3766
3767void LocationsBuilderX86::VisitShr(HShr* shr) {
3768  HandleShift(shr);
3769}
3770
3771void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
3772  HandleShift(shr);
3773}
3774
3775void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
3776  HandleShift(ushr);
3777}
3778
3779void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
3780  HandleShift(ushr);
3781}
3782
3783void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
3784  LocationSummary* locations =
3785      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3786  locations->SetOut(Location::RegisterLocation(EAX));
3787  InvokeRuntimeCallingConvention calling_convention;
3788  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3789  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3790}
3791
3792void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
3793  // Note: if heap poisoning is enabled, the entry point takes cares
3794  // of poisoning the reference.
3795  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3796                          instruction,
3797                          instruction->GetDexPc(),
3798                          nullptr);
3799  CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
3800  DCHECK(!codegen_->IsLeafMethod());
3801}
3802
3803void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
3804  LocationSummary* locations =
3805      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3806  locations->SetOut(Location::RegisterLocation(EAX));
3807  InvokeRuntimeCallingConvention calling_convention;
3808  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3809  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3810  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3811}
3812
3813void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
3814  InvokeRuntimeCallingConvention calling_convention;
3815  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
3816  // Note: if heap poisoning is enabled, the entry point takes cares
3817  // of poisoning the reference.
3818  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3819                          instruction,
3820                          instruction->GetDexPc(),
3821                          nullptr);
3822  CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>();
3823  DCHECK(!codegen_->IsLeafMethod());
3824}
3825
3826void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
3827  LocationSummary* locations =
3828      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3829  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3830  if (location.IsStackSlot()) {
3831    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3832  } else if (location.IsDoubleStackSlot()) {
3833    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3834  }
3835  locations->SetOut(location);
3836}
3837
3838void InstructionCodeGeneratorX86::VisitParameterValue(
3839    HParameterValue* instruction ATTRIBUTE_UNUSED) {
3840}
3841
3842void LocationsBuilderX86::VisitCurrentMethod(HCurrentMethod* instruction) {
3843  LocationSummary* locations =
3844      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3845  locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument));
3846}
3847
3848void InstructionCodeGeneratorX86::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
3849}
3850
3851void LocationsBuilderX86::VisitNot(HNot* not_) {
3852  LocationSummary* locations =
3853      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
3854  locations->SetInAt(0, Location::RequiresRegister());
3855  locations->SetOut(Location::SameAsFirstInput());
3856}
3857
3858void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
3859  LocationSummary* locations = not_->GetLocations();
3860  Location in = locations->InAt(0);
3861  Location out = locations->Out();
3862  DCHECK(in.Equals(out));
3863  switch (not_->GetResultType()) {
3864    case Primitive::kPrimInt:
3865      __ notl(out.AsRegister<Register>());
3866      break;
3867
3868    case Primitive::kPrimLong:
3869      __ notl(out.AsRegisterPairLow<Register>());
3870      __ notl(out.AsRegisterPairHigh<Register>());
3871      break;
3872
3873    default:
3874      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
3875  }
3876}
3877
3878void LocationsBuilderX86::VisitBooleanNot(HBooleanNot* bool_not) {
3879  LocationSummary* locations =
3880      new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
3881  locations->SetInAt(0, Location::RequiresRegister());
3882  locations->SetOut(Location::SameAsFirstInput());
3883}
3884
3885void InstructionCodeGeneratorX86::VisitBooleanNot(HBooleanNot* bool_not) {
3886  LocationSummary* locations = bool_not->GetLocations();
3887  Location in = locations->InAt(0);
3888  Location out = locations->Out();
3889  DCHECK(in.Equals(out));
3890  __ xorl(out.AsRegister<Register>(), Immediate(1));
3891}
3892
3893void LocationsBuilderX86::VisitCompare(HCompare* compare) {
3894  LocationSummary* locations =
3895      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
3896  switch (compare->InputAt(0)->GetType()) {
3897    case Primitive::kPrimLong: {
3898      locations->SetInAt(0, Location::RequiresRegister());
3899      locations->SetInAt(1, Location::Any());
3900      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3901      break;
3902    }
3903    case Primitive::kPrimFloat:
3904    case Primitive::kPrimDouble: {
3905      locations->SetInAt(0, Location::RequiresFpuRegister());
3906      locations->SetInAt(1, Location::RequiresFpuRegister());
3907      locations->SetOut(Location::RequiresRegister());
3908      break;
3909    }
3910    default:
3911      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3912  }
3913}
3914
3915void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
3916  LocationSummary* locations = compare->GetLocations();
3917  Register out = locations->Out().AsRegister<Register>();
3918  Location left = locations->InAt(0);
3919  Location right = locations->InAt(1);
3920
3921  NearLabel less, greater, done;
3922  switch (compare->InputAt(0)->GetType()) {
3923    case Primitive::kPrimLong: {
3924      Register left_low = left.AsRegisterPairLow<Register>();
3925      Register left_high = left.AsRegisterPairHigh<Register>();
3926      int32_t val_low = 0;
3927      int32_t val_high = 0;
3928      bool right_is_const = false;
3929
3930      if (right.IsConstant()) {
3931        DCHECK(right.GetConstant()->IsLongConstant());
3932        right_is_const = true;
3933        int64_t val = right.GetConstant()->AsLongConstant()->GetValue();
3934        val_low = Low32Bits(val);
3935        val_high = High32Bits(val);
3936      }
3937
3938      if (right.IsRegisterPair()) {
3939        __ cmpl(left_high, right.AsRegisterPairHigh<Register>());
3940      } else if (right.IsDoubleStackSlot()) {
3941        __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
3942      } else {
3943        DCHECK(right_is_const) << right;
3944        if (val_high == 0) {
3945          __ testl(left_high, left_high);
3946        } else {
3947          __ cmpl(left_high, Immediate(val_high));
3948        }
3949      }
3950      __ j(kLess, &less);  // Signed compare.
3951      __ j(kGreater, &greater);  // Signed compare.
3952      if (right.IsRegisterPair()) {
3953        __ cmpl(left_low, right.AsRegisterPairLow<Register>());
3954      } else if (right.IsDoubleStackSlot()) {
3955        __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
3956      } else {
3957        DCHECK(right_is_const) << right;
3958        if (val_low == 0) {
3959          __ testl(left_low, left_low);
3960        } else {
3961          __ cmpl(left_low, Immediate(val_low));
3962        }
3963      }
3964      break;
3965    }
3966    case Primitive::kPrimFloat: {
3967      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3968      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3969      break;
3970    }
3971    case Primitive::kPrimDouble: {
3972      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3973      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3974      break;
3975    }
3976    default:
3977      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3978  }
3979  __ movl(out, Immediate(0));
3980  __ j(kEqual, &done);
3981  __ j(kBelow, &less);  // kBelow is for CF (unsigned & floats).
3982
3983  __ Bind(&greater);
3984  __ movl(out, Immediate(1));
3985  __ jmp(&done);
3986
3987  __ Bind(&less);
3988  __ movl(out, Immediate(-1));
3989
3990  __ Bind(&done);
3991}
3992
3993void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
3994  LocationSummary* locations =
3995      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3996  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3997    locations->SetInAt(i, Location::Any());
3998  }
3999  locations->SetOut(Location::Any());
4000}
4001
4002void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) {
4003  LOG(FATAL) << "Unreachable";
4004}
4005
4006void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
4007  /*
4008   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
4009   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
4010   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
4011   */
4012  switch (kind) {
4013    case MemBarrierKind::kAnyAny: {
4014      __ mfence();
4015      break;
4016    }
4017    case MemBarrierKind::kAnyStore:
4018    case MemBarrierKind::kLoadAny:
4019    case MemBarrierKind::kStoreStore: {
4020      // nop
4021      break;
4022    }
4023    default:
4024      LOG(FATAL) << "Unexpected memory barrier " << kind;
4025  }
4026}
4027
4028HInvokeStaticOrDirect::DispatchInfo CodeGeneratorX86::GetSupportedInvokeStaticOrDirectDispatch(
4029      const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
4030      MethodReference target_method ATTRIBUTE_UNUSED) {
4031  switch (desired_dispatch_info.code_ptr_location) {
4032    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
4033    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
4034      // For direct code, we actually prefer to call via the code pointer from ArtMethod*.
4035      // (Though the direct CALL ptr16:32 is available for consideration).
4036      return HInvokeStaticOrDirect::DispatchInfo {
4037        desired_dispatch_info.method_load_kind,
4038        HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod,
4039        desired_dispatch_info.method_load_data,
4040        0u
4041      };
4042    default:
4043      return desired_dispatch_info;
4044  }
4045}
4046
4047Register CodeGeneratorX86::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke,
4048                                                                 Register temp) {
4049  DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u);
4050  Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4051  if (!invoke->GetLocations()->Intrinsified()) {
4052    return location.AsRegister<Register>();
4053  }
4054  // For intrinsics we allow any location, so it may be on the stack.
4055  if (!location.IsRegister()) {
4056    __ movl(temp, Address(ESP, location.GetStackIndex()));
4057    return temp;
4058  }
4059  // For register locations, check if the register was saved. If so, get it from the stack.
4060  // Note: There is a chance that the register was saved but not overwritten, so we could
4061  // save one load. However, since this is just an intrinsic slow path we prefer this
4062  // simple and more robust approach rather that trying to determine if that's the case.
4063  SlowPathCode* slow_path = GetCurrentSlowPath();
4064  DCHECK(slow_path != nullptr);  // For intrinsified invokes the call is emitted on the slow path.
4065  if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) {
4066    int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>());
4067    __ movl(temp, Address(ESP, stack_offset));
4068    return temp;
4069  }
4070  return location.AsRegister<Register>();
4071}
4072
4073void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) {
4074  Location callee_method = temp;  // For all kinds except kRecursive, callee will be in temp.
4075  switch (invoke->GetMethodLoadKind()) {
4076    case HInvokeStaticOrDirect::MethodLoadKind::kStringInit:
4077      // temp = thread->string_init_entrypoint
4078      __ fs()->movl(temp.AsRegister<Register>(), Address::Absolute(invoke->GetStringInitOffset()));
4079      break;
4080    case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
4081      callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4082      break;
4083    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
4084      __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress()));
4085      break;
4086    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
4087      __ movl(temp.AsRegister<Register>(), Immediate(0));  // Placeholder.
4088      method_patches_.emplace_back(invoke->GetTargetMethod());
4089      __ Bind(&method_patches_.back().label);  // Bind the label at the end of the "movl" insn.
4090      break;
4091    case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
4092      Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke,
4093                                                                temp.AsRegister<Register>());
4094      uint32_t offset = invoke->GetDexCacheArrayOffset();
4095      __ movl(temp.AsRegister<Register>(), Address(base_reg, kDummy32BitOffset));
4096      // Add the patch entry and bind its label at the end of the instruction.
4097      pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, offset);
4098      __ Bind(&pc_relative_dex_cache_patches_.back().label);
4099      break;
4100    }
4101    case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
4102      Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4103      Register method_reg;
4104      Register reg = temp.AsRegister<Register>();
4105      if (current_method.IsRegister()) {
4106        method_reg = current_method.AsRegister<Register>();
4107      } else {
4108        DCHECK(IsBaseline() || invoke->GetLocations()->Intrinsified());
4109        DCHECK(!current_method.IsValid());
4110        method_reg = reg;
4111        __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
4112      }
4113      // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_;
4114      __ movl(reg, Address(method_reg,
4115                           ArtMethod::DexCacheResolvedMethodsOffset(kX86PointerSize).Int32Value()));
4116      // temp = temp[index_in_cache]
4117      uint32_t index_in_cache = invoke->GetTargetMethod().dex_method_index;
4118      __ movl(reg, Address(reg, CodeGenerator::GetCachePointerOffset(index_in_cache)));
4119      break;
4120    }
4121  }
4122
4123  switch (invoke->GetCodePtrLocation()) {
4124    case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf:
4125      __ call(GetFrameEntryLabel());
4126      break;
4127    case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: {
4128      relative_call_patches_.emplace_back(invoke->GetTargetMethod());
4129      Label* label = &relative_call_patches_.back().label;
4130      __ call(label);  // Bind to the patch label, override at link time.
4131      __ Bind(label);  // Bind the label at the end of the "call" insn.
4132      break;
4133    }
4134    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
4135    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
4136      // Filtered out by GetSupportedInvokeStaticOrDirectDispatch().
4137      LOG(FATAL) << "Unsupported";
4138      UNREACHABLE();
4139    case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
4140      // (callee_method + offset_of_quick_compiled_code)()
4141      __ call(Address(callee_method.AsRegister<Register>(),
4142                      ArtMethod::EntryPointFromQuickCompiledCodeOffset(
4143                          kX86WordSize).Int32Value()));
4144      break;
4145  }
4146
4147  DCHECK(!IsLeafMethod());
4148}
4149
4150void CodeGeneratorX86::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) {
4151  Register temp = temp_in.AsRegister<Register>();
4152  uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
4153      invoke->GetVTableIndex(), kX86PointerSize).Uint32Value();
4154  LocationSummary* locations = invoke->GetLocations();
4155  Location receiver = locations->InAt(0);
4156  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4157  DCHECK(receiver.IsRegister());
4158  // /* HeapReference<Class> */ temp = receiver->klass_
4159  __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
4160  MaybeRecordImplicitNullCheck(invoke);
4161  // Instead of simply (possibly) unpoisoning `temp` here, we should
4162  // emit a read barrier for the previous class reference load.
4163  // However this is not required in practice, as this is an
4164  // intermediate/temporary reference and because the current
4165  // concurrent copying collector keeps the from-space memory
4166  // intact/accessible until the end of the marking phase (the
4167  // concurrent copying collector may not in the future).
4168  __ MaybeUnpoisonHeapReference(temp);
4169  // temp = temp->GetMethodAt(method_offset);
4170  __ movl(temp, Address(temp, method_offset));
4171  // call temp->GetEntryPoint();
4172  __ call(Address(
4173      temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
4174}
4175
4176void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
4177  DCHECK(linker_patches->empty());
4178  size_t size =
4179      method_patches_.size() +
4180      relative_call_patches_.size() +
4181      pc_relative_dex_cache_patches_.size();
4182  linker_patches->reserve(size);
4183  // The label points to the end of the "movl" insn but the literal offset for method
4184  // patch needs to point to the embedded constant which occupies the last 4 bytes.
4185  constexpr uint32_t kLabelPositionToLiteralOffsetAdjustment = 4u;
4186  for (const MethodPatchInfo<Label>& info : method_patches_) {
4187    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4188    linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset,
4189                                                       info.target_method.dex_file,
4190                                                       info.target_method.dex_method_index));
4191  }
4192  for (const MethodPatchInfo<Label>& info : relative_call_patches_) {
4193    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4194    linker_patches->push_back(LinkerPatch::RelativeCodePatch(literal_offset,
4195                                                             info.target_method.dex_file,
4196                                                             info.target_method.dex_method_index));
4197  }
4198  for (const PcRelativeDexCacheAccessInfo& info : pc_relative_dex_cache_patches_) {
4199    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4200    linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(literal_offset,
4201                                                              &info.target_dex_file,
4202                                                              GetMethodAddressOffset(),
4203                                                              info.element_offset));
4204  }
4205}
4206
4207void CodeGeneratorX86::MarkGCCard(Register temp,
4208                                  Register card,
4209                                  Register object,
4210                                  Register value,
4211                                  bool value_can_be_null) {
4212  NearLabel is_null;
4213  if (value_can_be_null) {
4214    __ testl(value, value);
4215    __ j(kEqual, &is_null);
4216  }
4217  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
4218  __ movl(temp, object);
4219  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
4220  __ movb(Address(temp, card, TIMES_1, 0),
4221          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
4222  if (value_can_be_null) {
4223    __ Bind(&is_null);
4224  }
4225}
4226
4227void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
4228  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
4229
4230  bool object_field_get_with_read_barrier =
4231      kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
4232  LocationSummary* locations =
4233      new (GetGraph()->GetArena()) LocationSummary(instruction,
4234                                                   kEmitCompilerReadBarrier ?
4235                                                       LocationSummary::kCallOnSlowPath :
4236                                                       LocationSummary::kNoCall);
4237  locations->SetInAt(0, Location::RequiresRegister());
4238
4239  if (Primitive::IsFloatingPointType(instruction->GetType())) {
4240    locations->SetOut(Location::RequiresFpuRegister());
4241  } else {
4242    // The output overlaps in case of long: we don't want the low move
4243    // to overwrite the object's location.  Likewise, in the case of
4244    // an object field get with read barriers enabled, we do not want
4245    // the move to overwrite the object's location, as we need it to emit
4246    // the read barrier.
4247    locations->SetOut(
4248        Location::RequiresRegister(),
4249        (object_field_get_with_read_barrier || instruction->GetType() == Primitive::kPrimLong) ?
4250            Location::kOutputOverlap :
4251            Location::kNoOutputOverlap);
4252  }
4253
4254  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
4255    // Long values can be loaded atomically into an XMM using movsd.
4256    // So we use an XMM register as a temp to achieve atomicity (first load the temp into the XMM
4257    // and then copy the XMM into the output 32bits at a time).
4258    locations->AddTemp(Location::RequiresFpuRegister());
4259  }
4260}
4261
4262void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
4263                                                 const FieldInfo& field_info) {
4264  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
4265
4266  LocationSummary* locations = instruction->GetLocations();
4267  Location base_loc = locations->InAt(0);
4268  Register base = base_loc.AsRegister<Register>();
4269  Location out = locations->Out();
4270  bool is_volatile = field_info.IsVolatile();
4271  Primitive::Type field_type = field_info.GetFieldType();
4272  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
4273
4274  switch (field_type) {
4275    case Primitive::kPrimBoolean: {
4276      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
4277      break;
4278    }
4279
4280    case Primitive::kPrimByte: {
4281      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
4282      break;
4283    }
4284
4285    case Primitive::kPrimShort: {
4286      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
4287      break;
4288    }
4289
4290    case Primitive::kPrimChar: {
4291      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
4292      break;
4293    }
4294
4295    case Primitive::kPrimInt:
4296    case Primitive::kPrimNot: {
4297      __ movl(out.AsRegister<Register>(), Address(base, offset));
4298      break;
4299    }
4300
4301    case Primitive::kPrimLong: {
4302      if (is_volatile) {
4303        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
4304        __ movsd(temp, Address(base, offset));
4305        codegen_->MaybeRecordImplicitNullCheck(instruction);
4306        __ movd(out.AsRegisterPairLow<Register>(), temp);
4307        __ psrlq(temp, Immediate(32));
4308        __ movd(out.AsRegisterPairHigh<Register>(), temp);
4309      } else {
4310        DCHECK_NE(base, out.AsRegisterPairLow<Register>());
4311        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
4312        codegen_->MaybeRecordImplicitNullCheck(instruction);
4313        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
4314      }
4315      break;
4316    }
4317
4318    case Primitive::kPrimFloat: {
4319      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
4320      break;
4321    }
4322
4323    case Primitive::kPrimDouble: {
4324      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
4325      break;
4326    }
4327
4328    case Primitive::kPrimVoid:
4329      LOG(FATAL) << "Unreachable type " << field_type;
4330      UNREACHABLE();
4331  }
4332
4333  // Longs are handled in the switch.
4334  if (field_type != Primitive::kPrimLong) {
4335    codegen_->MaybeRecordImplicitNullCheck(instruction);
4336  }
4337
4338  if (is_volatile) {
4339    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4340  }
4341
4342  if (field_type == Primitive::kPrimNot) {
4343    codegen_->MaybeGenerateReadBarrier(instruction, out, out, base_loc, offset);
4344  }
4345}
4346
4347void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
4348  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
4349
4350  LocationSummary* locations =
4351      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4352  locations->SetInAt(0, Location::RequiresRegister());
4353  bool is_volatile = field_info.IsVolatile();
4354  Primitive::Type field_type = field_info.GetFieldType();
4355  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
4356    || (field_type == Primitive::kPrimByte);
4357
4358  // The register allocator does not support multiple
4359  // inputs that die at entry with one in a specific register.
4360  if (is_byte_type) {
4361    // Ensure the value is in a byte register.
4362    locations->SetInAt(1, Location::RegisterLocation(EAX));
4363  } else if (Primitive::IsFloatingPointType(field_type)) {
4364    if (is_volatile && field_type == Primitive::kPrimDouble) {
4365      // In order to satisfy the semantics of volatile, this must be a single instruction store.
4366      locations->SetInAt(1, Location::RequiresFpuRegister());
4367    } else {
4368      locations->SetInAt(1, Location::FpuRegisterOrConstant(instruction->InputAt(1)));
4369    }
4370  } else if (is_volatile && field_type == Primitive::kPrimLong) {
4371    // In order to satisfy the semantics of volatile, this must be a single instruction store.
4372    locations->SetInAt(1, Location::RequiresRegister());
4373
4374    // 64bits value can be atomically written to an address with movsd and an XMM register.
4375    // We need two XMM registers because there's no easier way to (bit) copy a register pair
4376    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
4377    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
4378    // isolated cases when we need this it isn't worth adding the extra complexity.
4379    locations->AddTemp(Location::RequiresFpuRegister());
4380    locations->AddTemp(Location::RequiresFpuRegister());
4381  } else {
4382    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4383
4384    if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
4385      // Temporary registers for the write barrier.
4386      locations->AddTemp(Location::RequiresRegister());  // May be used for reference poisoning too.
4387      // Ensure the card is in a byte register.
4388      locations->AddTemp(Location::RegisterLocation(ECX));
4389    }
4390  }
4391}
4392
4393void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
4394                                                 const FieldInfo& field_info,
4395                                                 bool value_can_be_null) {
4396  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
4397
4398  LocationSummary* locations = instruction->GetLocations();
4399  Register base = locations->InAt(0).AsRegister<Register>();
4400  Location value = locations->InAt(1);
4401  bool is_volatile = field_info.IsVolatile();
4402  Primitive::Type field_type = field_info.GetFieldType();
4403  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
4404  bool needs_write_barrier =
4405      CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1));
4406
4407  if (is_volatile) {
4408    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
4409  }
4410
4411  bool maybe_record_implicit_null_check_done = false;
4412
4413  switch (field_type) {
4414    case Primitive::kPrimBoolean:
4415    case Primitive::kPrimByte: {
4416      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
4417      break;
4418    }
4419
4420    case Primitive::kPrimShort:
4421    case Primitive::kPrimChar: {
4422      if (value.IsConstant()) {
4423        int16_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4424        __ movw(Address(base, offset), Immediate(v));
4425      } else {
4426        __ movw(Address(base, offset), value.AsRegister<Register>());
4427      }
4428      break;
4429    }
4430
4431    case Primitive::kPrimInt:
4432    case Primitive::kPrimNot: {
4433      if (kPoisonHeapReferences && needs_write_barrier) {
4434        // Note that in the case where `value` is a null reference,
4435        // we do not enter this block, as the reference does not
4436        // need poisoning.
4437        DCHECK_EQ(field_type, Primitive::kPrimNot);
4438        Register temp = locations->GetTemp(0).AsRegister<Register>();
4439        __ movl(temp, value.AsRegister<Register>());
4440        __ PoisonHeapReference(temp);
4441        __ movl(Address(base, offset), temp);
4442      } else if (value.IsConstant()) {
4443        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4444        __ movl(Address(base, offset), Immediate(v));
4445      } else {
4446        __ movl(Address(base, offset), value.AsRegister<Register>());
4447      }
4448      break;
4449    }
4450
4451    case Primitive::kPrimLong: {
4452      if (is_volatile) {
4453        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
4454        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
4455        __ movd(temp1, value.AsRegisterPairLow<Register>());
4456        __ movd(temp2, value.AsRegisterPairHigh<Register>());
4457        __ punpckldq(temp1, temp2);
4458        __ movsd(Address(base, offset), temp1);
4459        codegen_->MaybeRecordImplicitNullCheck(instruction);
4460      } else if (value.IsConstant()) {
4461        int64_t v = CodeGenerator::GetInt64ValueOf(value.GetConstant());
4462        __ movl(Address(base, offset), Immediate(Low32Bits(v)));
4463        codegen_->MaybeRecordImplicitNullCheck(instruction);
4464        __ movl(Address(base, kX86WordSize + offset), Immediate(High32Bits(v)));
4465      } else {
4466        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
4467        codegen_->MaybeRecordImplicitNullCheck(instruction);
4468        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
4469      }
4470      maybe_record_implicit_null_check_done = true;
4471      break;
4472    }
4473
4474    case Primitive::kPrimFloat: {
4475      if (value.IsConstant()) {
4476        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4477        __ movl(Address(base, offset), Immediate(v));
4478      } else {
4479        __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
4480      }
4481      break;
4482    }
4483
4484    case Primitive::kPrimDouble: {
4485      if (value.IsConstant()) {
4486        int64_t v = CodeGenerator::GetInt64ValueOf(value.GetConstant());
4487        __ movl(Address(base, offset), Immediate(Low32Bits(v)));
4488        codegen_->MaybeRecordImplicitNullCheck(instruction);
4489        __ movl(Address(base, kX86WordSize + offset), Immediate(High32Bits(v)));
4490        maybe_record_implicit_null_check_done = true;
4491      } else {
4492        __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
4493      }
4494      break;
4495    }
4496
4497    case Primitive::kPrimVoid:
4498      LOG(FATAL) << "Unreachable type " << field_type;
4499      UNREACHABLE();
4500  }
4501
4502  if (!maybe_record_implicit_null_check_done) {
4503    codegen_->MaybeRecordImplicitNullCheck(instruction);
4504  }
4505
4506  if (needs_write_barrier) {
4507    Register temp = locations->GetTemp(0).AsRegister<Register>();
4508    Register card = locations->GetTemp(1).AsRegister<Register>();
4509    codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>(), value_can_be_null);
4510  }
4511
4512  if (is_volatile) {
4513    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
4514  }
4515}
4516
4517void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4518  HandleFieldGet(instruction, instruction->GetFieldInfo());
4519}
4520
4521void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4522  HandleFieldGet(instruction, instruction->GetFieldInfo());
4523}
4524
4525void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4526  HandleFieldSet(instruction, instruction->GetFieldInfo());
4527}
4528
4529void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4530  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4531}
4532
4533void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4534  HandleFieldSet(instruction, instruction->GetFieldInfo());
4535}
4536
4537void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4538  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4539}
4540
4541void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4542  HandleFieldGet(instruction, instruction->GetFieldInfo());
4543}
4544
4545void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4546  HandleFieldGet(instruction, instruction->GetFieldInfo());
4547}
4548
4549void LocationsBuilderX86::VisitUnresolvedInstanceFieldGet(
4550    HUnresolvedInstanceFieldGet* instruction) {
4551  FieldAccessCallingConventionX86 calling_convention;
4552  codegen_->CreateUnresolvedFieldLocationSummary(
4553      instruction, instruction->GetFieldType(), calling_convention);
4554}
4555
4556void InstructionCodeGeneratorX86::VisitUnresolvedInstanceFieldGet(
4557    HUnresolvedInstanceFieldGet* instruction) {
4558  FieldAccessCallingConventionX86 calling_convention;
4559  codegen_->GenerateUnresolvedFieldAccess(instruction,
4560                                          instruction->GetFieldType(),
4561                                          instruction->GetFieldIndex(),
4562                                          instruction->GetDexPc(),
4563                                          calling_convention);
4564}
4565
4566void LocationsBuilderX86::VisitUnresolvedInstanceFieldSet(
4567    HUnresolvedInstanceFieldSet* instruction) {
4568  FieldAccessCallingConventionX86 calling_convention;
4569  codegen_->CreateUnresolvedFieldLocationSummary(
4570      instruction, instruction->GetFieldType(), calling_convention);
4571}
4572
4573void InstructionCodeGeneratorX86::VisitUnresolvedInstanceFieldSet(
4574    HUnresolvedInstanceFieldSet* instruction) {
4575  FieldAccessCallingConventionX86 calling_convention;
4576  codegen_->GenerateUnresolvedFieldAccess(instruction,
4577                                          instruction->GetFieldType(),
4578                                          instruction->GetFieldIndex(),
4579                                          instruction->GetDexPc(),
4580                                          calling_convention);
4581}
4582
4583void LocationsBuilderX86::VisitUnresolvedStaticFieldGet(
4584    HUnresolvedStaticFieldGet* instruction) {
4585  FieldAccessCallingConventionX86 calling_convention;
4586  codegen_->CreateUnresolvedFieldLocationSummary(
4587      instruction, instruction->GetFieldType(), calling_convention);
4588}
4589
4590void InstructionCodeGeneratorX86::VisitUnresolvedStaticFieldGet(
4591    HUnresolvedStaticFieldGet* instruction) {
4592  FieldAccessCallingConventionX86 calling_convention;
4593  codegen_->GenerateUnresolvedFieldAccess(instruction,
4594                                          instruction->GetFieldType(),
4595                                          instruction->GetFieldIndex(),
4596                                          instruction->GetDexPc(),
4597                                          calling_convention);
4598}
4599
4600void LocationsBuilderX86::VisitUnresolvedStaticFieldSet(
4601    HUnresolvedStaticFieldSet* instruction) {
4602  FieldAccessCallingConventionX86 calling_convention;
4603  codegen_->CreateUnresolvedFieldLocationSummary(
4604      instruction, instruction->GetFieldType(), calling_convention);
4605}
4606
4607void InstructionCodeGeneratorX86::VisitUnresolvedStaticFieldSet(
4608    HUnresolvedStaticFieldSet* instruction) {
4609  FieldAccessCallingConventionX86 calling_convention;
4610  codegen_->GenerateUnresolvedFieldAccess(instruction,
4611                                          instruction->GetFieldType(),
4612                                          instruction->GetFieldIndex(),
4613                                          instruction->GetDexPc(),
4614                                          calling_convention);
4615}
4616
4617void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
4618  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
4619      ? LocationSummary::kCallOnSlowPath
4620      : LocationSummary::kNoCall;
4621  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4622  Location loc = codegen_->IsImplicitNullCheckAllowed(instruction)
4623      ? Location::RequiresRegister()
4624      : Location::Any();
4625  locations->SetInAt(0, loc);
4626  if (instruction->HasUses()) {
4627    locations->SetOut(Location::SameAsFirstInput());
4628  }
4629}
4630
4631void InstructionCodeGeneratorX86::GenerateImplicitNullCheck(HNullCheck* instruction) {
4632  if (codegen_->CanMoveNullCheckToUser(instruction)) {
4633    return;
4634  }
4635  LocationSummary* locations = instruction->GetLocations();
4636  Location obj = locations->InAt(0);
4637
4638  __ testl(EAX, Address(obj.AsRegister<Register>(), 0));
4639  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4640}
4641
4642void InstructionCodeGeneratorX86::GenerateExplicitNullCheck(HNullCheck* instruction) {
4643  SlowPathCode* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
4644  codegen_->AddSlowPath(slow_path);
4645
4646  LocationSummary* locations = instruction->GetLocations();
4647  Location obj = locations->InAt(0);
4648
4649  if (obj.IsRegister()) {
4650    __ testl(obj.AsRegister<Register>(), obj.AsRegister<Register>());
4651  } else if (obj.IsStackSlot()) {
4652    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
4653  } else {
4654    DCHECK(obj.IsConstant()) << obj;
4655    DCHECK(obj.GetConstant()->IsNullConstant());
4656    __ jmp(slow_path->GetEntryLabel());
4657    return;
4658  }
4659  __ j(kEqual, slow_path->GetEntryLabel());
4660}
4661
4662void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
4663  if (codegen_->IsImplicitNullCheckAllowed(instruction)) {
4664    GenerateImplicitNullCheck(instruction);
4665  } else {
4666    GenerateExplicitNullCheck(instruction);
4667  }
4668}
4669
4670void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
4671  bool object_array_get_with_read_barrier =
4672      kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
4673  LocationSummary* locations =
4674      new (GetGraph()->GetArena()) LocationSummary(instruction,
4675                                                   object_array_get_with_read_barrier ?
4676                                                       LocationSummary::kCallOnSlowPath :
4677                                                       LocationSummary::kNoCall);
4678  locations->SetInAt(0, Location::RequiresRegister());
4679  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4680  if (Primitive::IsFloatingPointType(instruction->GetType())) {
4681    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
4682  } else {
4683    // The output overlaps in case of long: we don't want the low move
4684    // to overwrite the array's location.  Likewise, in the case of an
4685    // object array get with read barriers enabled, we do not want the
4686    // move to overwrite the array's location, as we need it to emit
4687    // the read barrier.
4688    locations->SetOut(
4689        Location::RequiresRegister(),
4690        (instruction->GetType() == Primitive::kPrimLong || object_array_get_with_read_barrier) ?
4691            Location::kOutputOverlap :
4692            Location::kNoOutputOverlap);
4693  }
4694}
4695
4696void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
4697  LocationSummary* locations = instruction->GetLocations();
4698  Location obj_loc = locations->InAt(0);
4699  Register obj = obj_loc.AsRegister<Register>();
4700  Location index = locations->InAt(1);
4701
4702  Primitive::Type type = instruction->GetType();
4703  switch (type) {
4704    case Primitive::kPrimBoolean: {
4705      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
4706      Register out = locations->Out().AsRegister<Register>();
4707      if (index.IsConstant()) {
4708        __ movzxb(out, Address(obj,
4709            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
4710      } else {
4711        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
4712      }
4713      break;
4714    }
4715
4716    case Primitive::kPrimByte: {
4717      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
4718      Register out = locations->Out().AsRegister<Register>();
4719      if (index.IsConstant()) {
4720        __ movsxb(out, Address(obj,
4721            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
4722      } else {
4723        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
4724      }
4725      break;
4726    }
4727
4728    case Primitive::kPrimShort: {
4729      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
4730      Register out = locations->Out().AsRegister<Register>();
4731      if (index.IsConstant()) {
4732        __ movsxw(out, Address(obj,
4733            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
4734      } else {
4735        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
4736      }
4737      break;
4738    }
4739
4740    case Primitive::kPrimChar: {
4741      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
4742      Register out = locations->Out().AsRegister<Register>();
4743      if (index.IsConstant()) {
4744        __ movzxw(out, Address(obj,
4745            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
4746      } else {
4747        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
4748      }
4749      break;
4750    }
4751
4752    case Primitive::kPrimInt:
4753    case Primitive::kPrimNot: {
4754      static_assert(
4755          sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
4756          "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
4757      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4758      Register out = locations->Out().AsRegister<Register>();
4759      if (index.IsConstant()) {
4760        __ movl(out, Address(obj,
4761            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
4762      } else {
4763        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
4764      }
4765      break;
4766    }
4767
4768    case Primitive::kPrimLong: {
4769      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
4770      Location out = locations->Out();
4771      DCHECK_NE(obj, out.AsRegisterPairLow<Register>());
4772      if (index.IsConstant()) {
4773        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4774        __ movl(out.AsRegisterPairLow<Register>(), Address(obj, offset));
4775        codegen_->MaybeRecordImplicitNullCheck(instruction);
4776        __ movl(out.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
4777      } else {
4778        __ movl(out.AsRegisterPairLow<Register>(),
4779                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
4780        codegen_->MaybeRecordImplicitNullCheck(instruction);
4781        __ movl(out.AsRegisterPairHigh<Register>(),
4782                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
4783      }
4784      break;
4785    }
4786
4787    case Primitive::kPrimFloat: {
4788      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
4789      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
4790      if (index.IsConstant()) {
4791        __ movss(out, Address(obj,
4792            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
4793      } else {
4794        __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
4795      }
4796      break;
4797    }
4798
4799    case Primitive::kPrimDouble: {
4800      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
4801      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
4802      if (index.IsConstant()) {
4803        __ movsd(out, Address(obj,
4804            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
4805      } else {
4806        __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
4807      }
4808      break;
4809    }
4810
4811    case Primitive::kPrimVoid:
4812      LOG(FATAL) << "Unreachable type " << type;
4813      UNREACHABLE();
4814  }
4815
4816  if (type != Primitive::kPrimLong) {
4817    codegen_->MaybeRecordImplicitNullCheck(instruction);
4818  }
4819
4820  if (type == Primitive::kPrimNot) {
4821    static_assert(
4822        sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
4823        "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
4824    uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4825    Location out = locations->Out();
4826    if (index.IsConstant()) {
4827      uint32_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4828      codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, offset);
4829    } else {
4830      codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, data_offset, index);
4831    }
4832  }
4833}
4834
4835void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
4836  // This location builder might end up asking to up to four registers, which is
4837  // not currently possible for baseline. The situation in which we need four
4838  // registers cannot be met by baseline though, because it has not run any
4839  // optimization.
4840
4841  Primitive::Type value_type = instruction->GetComponentType();
4842
4843  bool needs_write_barrier =
4844      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
4845  bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
4846  bool object_array_set_with_read_barrier =
4847      kEmitCompilerReadBarrier && (value_type == Primitive::kPrimNot);
4848
4849  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4850      instruction,
4851      (may_need_runtime_call_for_type_check || object_array_set_with_read_barrier) ?
4852          LocationSummary::kCallOnSlowPath :
4853          LocationSummary::kNoCall);
4854
4855  bool is_byte_type = (value_type == Primitive::kPrimBoolean)
4856      || (value_type == Primitive::kPrimByte);
4857  // We need the inputs to be different than the output in case of long operation.
4858  // In case of a byte operation, the register allocator does not support multiple
4859  // inputs that die at entry with one in a specific register.
4860  locations->SetInAt(0, Location::RequiresRegister());
4861  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4862  if (is_byte_type) {
4863    // Ensure the value is in a byte register.
4864    locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
4865  } else if (Primitive::IsFloatingPointType(value_type)) {
4866    locations->SetInAt(2, Location::FpuRegisterOrConstant(instruction->InputAt(2)));
4867  } else {
4868    locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
4869  }
4870  if (needs_write_barrier) {
4871    // Temporary registers for the write barrier.
4872    locations->AddTemp(Location::RequiresRegister());  // Possibly used for ref. poisoning too.
4873    // Ensure the card is in a byte register.
4874    locations->AddTemp(Location::RegisterLocation(ECX));
4875  }
4876}
4877
4878void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
4879  LocationSummary* locations = instruction->GetLocations();
4880  Location array_loc = locations->InAt(0);
4881  Register array = array_loc.AsRegister<Register>();
4882  Location index = locations->InAt(1);
4883  Location value = locations->InAt(2);
4884  Primitive::Type value_type = instruction->GetComponentType();
4885  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4886  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
4887  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
4888  bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
4889  bool needs_write_barrier =
4890      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
4891
4892  switch (value_type) {
4893    case Primitive::kPrimBoolean:
4894    case Primitive::kPrimByte: {
4895      uint32_t offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
4896      Address address = index.IsConstant()
4897          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + offset)
4898          : Address(array, index.AsRegister<Register>(), TIMES_1, offset);
4899      if (value.IsRegister()) {
4900        __ movb(address, value.AsRegister<ByteRegister>());
4901      } else {
4902        __ movb(address, Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4903      }
4904      codegen_->MaybeRecordImplicitNullCheck(instruction);
4905      break;
4906    }
4907
4908    case Primitive::kPrimShort:
4909    case Primitive::kPrimChar: {
4910      uint32_t offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
4911      Address address = index.IsConstant()
4912          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + offset)
4913          : Address(array, index.AsRegister<Register>(), TIMES_2, offset);
4914      if (value.IsRegister()) {
4915        __ movw(address, value.AsRegister<Register>());
4916      } else {
4917        __ movw(address, Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4918      }
4919      codegen_->MaybeRecordImplicitNullCheck(instruction);
4920      break;
4921    }
4922
4923    case Primitive::kPrimNot: {
4924      uint32_t offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4925      Address address = index.IsConstant()
4926          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
4927          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
4928
4929      if (!value.IsRegister()) {
4930        // Just setting null.
4931        DCHECK(instruction->InputAt(2)->IsNullConstant());
4932        DCHECK(value.IsConstant()) << value;
4933        __ movl(address, Immediate(0));
4934        codegen_->MaybeRecordImplicitNullCheck(instruction);
4935        DCHECK(!needs_write_barrier);
4936        DCHECK(!may_need_runtime_call_for_type_check);
4937        break;
4938      }
4939
4940      DCHECK(needs_write_barrier);
4941      Register register_value = value.AsRegister<Register>();
4942      NearLabel done, not_null, do_put;
4943      SlowPathCode* slow_path = nullptr;
4944      Register temp = locations->GetTemp(0).AsRegister<Register>();
4945      if (may_need_runtime_call_for_type_check) {
4946        slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathX86(instruction);
4947        codegen_->AddSlowPath(slow_path);
4948        if (instruction->GetValueCanBeNull()) {
4949          __ testl(register_value, register_value);
4950          __ j(kNotEqual, &not_null);
4951          __ movl(address, Immediate(0));
4952          codegen_->MaybeRecordImplicitNullCheck(instruction);
4953          __ jmp(&done);
4954          __ Bind(&not_null);
4955        }
4956
4957        if (kEmitCompilerReadBarrier) {
4958          // When read barriers are enabled, the type checking
4959          // instrumentation requires two read barriers:
4960          //
4961          //   __ movl(temp2, temp);
4962          //   // /* HeapReference<Class> */ temp = temp->component_type_
4963          //   __ movl(temp, Address(temp, component_offset));
4964          //   codegen_->GenerateReadBarrier(
4965          //       instruction, temp_loc, temp_loc, temp2_loc, component_offset);
4966          //
4967          //   // /* HeapReference<Class> */ temp2 = register_value->klass_
4968          //   __ movl(temp2, Address(register_value, class_offset));
4969          //   codegen_->GenerateReadBarrier(
4970          //       instruction, temp2_loc, temp2_loc, value, class_offset, temp_loc);
4971          //
4972          //   __ cmpl(temp, temp2);
4973          //
4974          // However, the second read barrier may trash `temp`, as it
4975          // is a temporary register, and as such would not be saved
4976          // along with live registers before calling the runtime (nor
4977          // restored afterwards).  So in this case, we bail out and
4978          // delegate the work to the array set slow path.
4979          //
4980          // TODO: Extend the register allocator to support a new
4981          // "(locally) live temp" location so as to avoid always
4982          // going into the slow path when read barriers are enabled.
4983          __ jmp(slow_path->GetEntryLabel());
4984        } else {
4985          // /* HeapReference<Class> */ temp = array->klass_
4986          __ movl(temp, Address(array, class_offset));
4987          codegen_->MaybeRecordImplicitNullCheck(instruction);
4988          __ MaybeUnpoisonHeapReference(temp);
4989
4990          // /* HeapReference<Class> */ temp = temp->component_type_
4991          __ movl(temp, Address(temp, component_offset));
4992          // If heap poisoning is enabled, no need to unpoison `temp`
4993          // nor the object reference in `register_value->klass`, as
4994          // we are comparing two poisoned references.
4995          __ cmpl(temp, Address(register_value, class_offset));
4996
4997          if (instruction->StaticTypeOfArrayIsObjectArray()) {
4998            __ j(kEqual, &do_put);
4999            // If heap poisoning is enabled, the `temp` reference has
5000            // not been unpoisoned yet; unpoison it now.
5001            __ MaybeUnpoisonHeapReference(temp);
5002
5003            // /* HeapReference<Class> */ temp = temp->super_class_
5004            __ movl(temp, Address(temp, super_offset));
5005            // If heap poisoning is enabled, no need to unpoison
5006            // `temp`, as we are comparing against null below.
5007            __ testl(temp, temp);
5008            __ j(kNotEqual, slow_path->GetEntryLabel());
5009            __ Bind(&do_put);
5010          } else {
5011            __ j(kNotEqual, slow_path->GetEntryLabel());
5012          }
5013        }
5014      }
5015
5016      if (kPoisonHeapReferences) {
5017        __ movl(temp, register_value);
5018        __ PoisonHeapReference(temp);
5019        __ movl(address, temp);
5020      } else {
5021        __ movl(address, register_value);
5022      }
5023      if (!may_need_runtime_call_for_type_check) {
5024        codegen_->MaybeRecordImplicitNullCheck(instruction);
5025      }
5026
5027      Register card = locations->GetTemp(1).AsRegister<Register>();
5028      codegen_->MarkGCCard(
5029          temp, card, array, value.AsRegister<Register>(), instruction->GetValueCanBeNull());
5030      __ Bind(&done);
5031
5032      if (slow_path != nullptr) {
5033        __ Bind(slow_path->GetExitLabel());
5034      }
5035
5036      break;
5037    }
5038
5039    case Primitive::kPrimInt: {
5040      uint32_t offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
5041      Address address = index.IsConstant()
5042          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5043          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5044      if (value.IsRegister()) {
5045        __ movl(address, value.AsRegister<Register>());
5046      } else {
5047        DCHECK(value.IsConstant()) << value;
5048        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
5049        __ movl(address, Immediate(v));
5050      }
5051      codegen_->MaybeRecordImplicitNullCheck(instruction);
5052      break;
5053    }
5054
5055    case Primitive::kPrimLong: {
5056      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
5057      if (index.IsConstant()) {
5058        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
5059        if (value.IsRegisterPair()) {
5060          __ movl(Address(array, offset), value.AsRegisterPairLow<Register>());
5061          codegen_->MaybeRecordImplicitNullCheck(instruction);
5062          __ movl(Address(array, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
5063        } else {
5064          DCHECK(value.IsConstant());
5065          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
5066          __ movl(Address(array, offset), Immediate(Low32Bits(val)));
5067          codegen_->MaybeRecordImplicitNullCheck(instruction);
5068          __ movl(Address(array, offset + kX86WordSize), Immediate(High32Bits(val)));
5069        }
5070      } else {
5071        if (value.IsRegisterPair()) {
5072          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset),
5073                  value.AsRegisterPairLow<Register>());
5074          codegen_->MaybeRecordImplicitNullCheck(instruction);
5075          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
5076                  value.AsRegisterPairHigh<Register>());
5077        } else {
5078          DCHECK(value.IsConstant());
5079          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
5080          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset),
5081                  Immediate(Low32Bits(val)));
5082          codegen_->MaybeRecordImplicitNullCheck(instruction);
5083          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
5084                  Immediate(High32Bits(val)));
5085        }
5086      }
5087      break;
5088    }
5089
5090    case Primitive::kPrimFloat: {
5091      uint32_t offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
5092      Address address = index.IsConstant()
5093          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5094          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5095      if (value.IsFpuRegister()) {
5096        __ movss(address, value.AsFpuRegister<XmmRegister>());
5097      } else {
5098        DCHECK(value.IsConstant());
5099        int32_t v = bit_cast<int32_t, float>(value.GetConstant()->AsFloatConstant()->GetValue());
5100        __ movl(address, Immediate(v));
5101      }
5102      codegen_->MaybeRecordImplicitNullCheck(instruction);
5103      break;
5104    }
5105
5106    case Primitive::kPrimDouble: {
5107      uint32_t offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
5108      Address address = index.IsConstant()
5109          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + offset)
5110          : Address(array, index.AsRegister<Register>(), TIMES_8, offset);
5111      if (value.IsFpuRegister()) {
5112        __ movsd(address, value.AsFpuRegister<XmmRegister>());
5113      } else {
5114        DCHECK(value.IsConstant());
5115        Address address_hi = index.IsConstant() ?
5116            Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) +
5117                           offset + kX86WordSize) :
5118            Address(array, index.AsRegister<Register>(), TIMES_8, offset + kX86WordSize);
5119        int64_t v = bit_cast<int64_t, double>(value.GetConstant()->AsDoubleConstant()->GetValue());
5120        __ movl(address, Immediate(Low32Bits(v)));
5121        codegen_->MaybeRecordImplicitNullCheck(instruction);
5122        __ movl(address_hi, Immediate(High32Bits(v)));
5123      }
5124      break;
5125    }
5126
5127    case Primitive::kPrimVoid:
5128      LOG(FATAL) << "Unreachable type " << instruction->GetType();
5129      UNREACHABLE();
5130  }
5131}
5132
5133void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
5134  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
5135  locations->SetInAt(0, Location::RequiresRegister());
5136  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
5137}
5138
5139void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
5140  LocationSummary* locations = instruction->GetLocations();
5141  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
5142  Register obj = locations->InAt(0).AsRegister<Register>();
5143  Register out = locations->Out().AsRegister<Register>();
5144  __ movl(out, Address(obj, offset));
5145  codegen_->MaybeRecordImplicitNullCheck(instruction);
5146}
5147
5148void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
5149  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
5150      ? LocationSummary::kCallOnSlowPath
5151      : LocationSummary::kNoCall;
5152  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5153  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
5154  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
5155  if (instruction->HasUses()) {
5156    locations->SetOut(Location::SameAsFirstInput());
5157  }
5158}
5159
5160void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
5161  LocationSummary* locations = instruction->GetLocations();
5162  Location index_loc = locations->InAt(0);
5163  Location length_loc = locations->InAt(1);
5164  SlowPathCode* slow_path =
5165    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(instruction);
5166
5167  if (length_loc.IsConstant()) {
5168    int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
5169    if (index_loc.IsConstant()) {
5170      // BCE will remove the bounds check if we are guarenteed to pass.
5171      int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
5172      if (index < 0 || index >= length) {
5173        codegen_->AddSlowPath(slow_path);
5174        __ jmp(slow_path->GetEntryLabel());
5175      } else {
5176        // Some optimization after BCE may have generated this, and we should not
5177        // generate a bounds check if it is a valid range.
5178      }
5179      return;
5180    }
5181
5182    // We have to reverse the jump condition because the length is the constant.
5183    Register index_reg = index_loc.AsRegister<Register>();
5184    __ cmpl(index_reg, Immediate(length));
5185    codegen_->AddSlowPath(slow_path);
5186    __ j(kAboveEqual, slow_path->GetEntryLabel());
5187  } else {
5188    Register length = length_loc.AsRegister<Register>();
5189    if (index_loc.IsConstant()) {
5190      int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
5191      __ cmpl(length, Immediate(value));
5192    } else {
5193      __ cmpl(length, index_loc.AsRegister<Register>());
5194    }
5195    codegen_->AddSlowPath(slow_path);
5196    __ j(kBelowEqual, slow_path->GetEntryLabel());
5197  }
5198}
5199
5200void LocationsBuilderX86::VisitTemporary(HTemporary* temp) {
5201  temp->SetLocations(nullptr);
5202}
5203
5204void InstructionCodeGeneratorX86::VisitTemporary(HTemporary* temp ATTRIBUTE_UNUSED) {
5205  // Nothing to do, this is driven by the code generator.
5206}
5207
5208void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) {
5209  LOG(FATAL) << "Unreachable";
5210}
5211
5212void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
5213  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
5214}
5215
5216void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
5217  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
5218}
5219
5220void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
5221  HBasicBlock* block = instruction->GetBlock();
5222  if (block->GetLoopInformation() != nullptr) {
5223    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
5224    // The back edge will generate the suspend check.
5225    return;
5226  }
5227  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
5228    // The goto will generate the suspend check.
5229    return;
5230  }
5231  GenerateSuspendCheck(instruction, nullptr);
5232}
5233
5234void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
5235                                                       HBasicBlock* successor) {
5236  SuspendCheckSlowPathX86* slow_path =
5237      down_cast<SuspendCheckSlowPathX86*>(instruction->GetSlowPath());
5238  if (slow_path == nullptr) {
5239    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
5240    instruction->SetSlowPath(slow_path);
5241    codegen_->AddSlowPath(slow_path);
5242    if (successor != nullptr) {
5243      DCHECK(successor->IsLoopHeader());
5244      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
5245    }
5246  } else {
5247    DCHECK_EQ(slow_path->GetSuccessor(), successor);
5248  }
5249
5250  __ fs()->cmpw(Address::Absolute(
5251      Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), Immediate(0));
5252  if (successor == nullptr) {
5253    __ j(kNotEqual, slow_path->GetEntryLabel());
5254    __ Bind(slow_path->GetReturnLabel());
5255  } else {
5256    __ j(kEqual, codegen_->GetLabelOf(successor));
5257    __ jmp(slow_path->GetEntryLabel());
5258  }
5259}
5260
5261X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
5262  return codegen_->GetAssembler();
5263}
5264
5265void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) {
5266  ScratchRegisterScope ensure_scratch(
5267      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5268  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5269  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5270  __ movl(temp_reg, Address(ESP, src + stack_offset));
5271  __ movl(Address(ESP, dst + stack_offset), temp_reg);
5272}
5273
5274void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) {
5275  ScratchRegisterScope ensure_scratch(
5276      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5277  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5278  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5279  __ movl(temp_reg, Address(ESP, src + stack_offset));
5280  __ movl(Address(ESP, dst + stack_offset), temp_reg);
5281  __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize));
5282  __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg);
5283}
5284
5285void ParallelMoveResolverX86::EmitMove(size_t index) {
5286  MoveOperands* move = moves_[index];
5287  Location source = move->GetSource();
5288  Location destination = move->GetDestination();
5289
5290  if (source.IsRegister()) {
5291    if (destination.IsRegister()) {
5292      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5293    } else {
5294      DCHECK(destination.IsStackSlot());
5295      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
5296    }
5297  } else if (source.IsFpuRegister()) {
5298    if (destination.IsFpuRegister()) {
5299      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5300    } else if (destination.IsStackSlot()) {
5301      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
5302    } else {
5303      DCHECK(destination.IsDoubleStackSlot());
5304      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
5305    }
5306  } else if (source.IsStackSlot()) {
5307    if (destination.IsRegister()) {
5308      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
5309    } else if (destination.IsFpuRegister()) {
5310      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
5311    } else {
5312      DCHECK(destination.IsStackSlot());
5313      MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex());
5314    }
5315  } else if (source.IsDoubleStackSlot()) {
5316    if (destination.IsFpuRegister()) {
5317      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
5318    } else {
5319      DCHECK(destination.IsDoubleStackSlot()) << destination;
5320      MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex());
5321    }
5322  } else if (source.IsConstant()) {
5323    HConstant* constant = source.GetConstant();
5324    if (constant->IsIntConstant() || constant->IsNullConstant()) {
5325      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
5326      if (destination.IsRegister()) {
5327        if (value == 0) {
5328          __ xorl(destination.AsRegister<Register>(), destination.AsRegister<Register>());
5329        } else {
5330          __ movl(destination.AsRegister<Register>(), Immediate(value));
5331        }
5332      } else {
5333        DCHECK(destination.IsStackSlot()) << destination;
5334        __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
5335      }
5336    } else if (constant->IsFloatConstant()) {
5337      float fp_value = constant->AsFloatConstant()->GetValue();
5338      int32_t value = bit_cast<int32_t, float>(fp_value);
5339      Immediate imm(value);
5340      if (destination.IsFpuRegister()) {
5341        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
5342        if (value == 0) {
5343          // Easy handling of 0.0.
5344          __ xorps(dest, dest);
5345        } else {
5346          ScratchRegisterScope ensure_scratch(
5347              this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5348          Register temp = static_cast<Register>(ensure_scratch.GetRegister());
5349          __ movl(temp, Immediate(value));
5350          __ movd(dest, temp);
5351        }
5352      } else {
5353        DCHECK(destination.IsStackSlot()) << destination;
5354        __ movl(Address(ESP, destination.GetStackIndex()), imm);
5355      }
5356    } else if (constant->IsLongConstant()) {
5357      int64_t value = constant->AsLongConstant()->GetValue();
5358      int32_t low_value = Low32Bits(value);
5359      int32_t high_value = High32Bits(value);
5360      Immediate low(low_value);
5361      Immediate high(high_value);
5362      if (destination.IsDoubleStackSlot()) {
5363        __ movl(Address(ESP, destination.GetStackIndex()), low);
5364        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
5365      } else {
5366        __ movl(destination.AsRegisterPairLow<Register>(), low);
5367        __ movl(destination.AsRegisterPairHigh<Register>(), high);
5368      }
5369    } else {
5370      DCHECK(constant->IsDoubleConstant());
5371      double dbl_value = constant->AsDoubleConstant()->GetValue();
5372      int64_t value = bit_cast<int64_t, double>(dbl_value);
5373      int32_t low_value = Low32Bits(value);
5374      int32_t high_value = High32Bits(value);
5375      Immediate low(low_value);
5376      Immediate high(high_value);
5377      if (destination.IsFpuRegister()) {
5378        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
5379        if (value == 0) {
5380          // Easy handling of 0.0.
5381          __ xorpd(dest, dest);
5382        } else {
5383          __ pushl(high);
5384          __ pushl(low);
5385          __ movsd(dest, Address(ESP, 0));
5386          __ addl(ESP, Immediate(8));
5387        }
5388      } else {
5389        DCHECK(destination.IsDoubleStackSlot()) << destination;
5390        __ movl(Address(ESP, destination.GetStackIndex()), low);
5391        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
5392      }
5393    }
5394  } else {
5395    LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
5396  }
5397}
5398
5399void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
5400  Register suggested_scratch = reg == EAX ? EBX : EAX;
5401  ScratchRegisterScope ensure_scratch(
5402      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
5403
5404  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5405  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
5406  __ movl(Address(ESP, mem + stack_offset), reg);
5407  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
5408}
5409
5410void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) {
5411  ScratchRegisterScope ensure_scratch(
5412      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5413
5414  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5415  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5416  __ movl(temp_reg, Address(ESP, mem + stack_offset));
5417  __ movss(Address(ESP, mem + stack_offset), reg);
5418  __ movd(reg, temp_reg);
5419}
5420
5421void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
5422  ScratchRegisterScope ensure_scratch1(
5423      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5424
5425  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
5426  ScratchRegisterScope ensure_scratch2(
5427      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
5428
5429  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
5430  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
5431  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
5432  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
5433  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
5434  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
5435}
5436
5437void ParallelMoveResolverX86::EmitSwap(size_t index) {
5438  MoveOperands* move = moves_[index];
5439  Location source = move->GetSource();
5440  Location destination = move->GetDestination();
5441
5442  if (source.IsRegister() && destination.IsRegister()) {
5443    // Use XOR swap algorithm to avoid serializing XCHG instruction or using a temporary.
5444    DCHECK_NE(destination.AsRegister<Register>(), source.AsRegister<Register>());
5445    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5446    __ xorl(source.AsRegister<Register>(), destination.AsRegister<Register>());
5447    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5448  } else if (source.IsRegister() && destination.IsStackSlot()) {
5449    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
5450  } else if (source.IsStackSlot() && destination.IsRegister()) {
5451    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
5452  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
5453    Exchange(destination.GetStackIndex(), source.GetStackIndex());
5454  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
5455    // Use XOR Swap algorithm to avoid a temporary.
5456    DCHECK_NE(source.reg(), destination.reg());
5457    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5458    __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
5459    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5460  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
5461    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
5462  } else if (destination.IsFpuRegister() && source.IsStackSlot()) {
5463    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
5464  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
5465    // Take advantage of the 16 bytes in the XMM register.
5466    XmmRegister reg = source.AsFpuRegister<XmmRegister>();
5467    Address stack(ESP, destination.GetStackIndex());
5468    // Load the double into the high doubleword.
5469    __ movhpd(reg, stack);
5470
5471    // Store the low double into the destination.
5472    __ movsd(stack, reg);
5473
5474    // Move the high double to the low double.
5475    __ psrldq(reg, Immediate(8));
5476  } else if (destination.IsFpuRegister() && source.IsDoubleStackSlot()) {
5477    // Take advantage of the 16 bytes in the XMM register.
5478    XmmRegister reg = destination.AsFpuRegister<XmmRegister>();
5479    Address stack(ESP, source.GetStackIndex());
5480    // Load the double into the high doubleword.
5481    __ movhpd(reg, stack);
5482
5483    // Store the low double into the destination.
5484    __ movsd(stack, reg);
5485
5486    // Move the high double to the low double.
5487    __ psrldq(reg, Immediate(8));
5488  } else if (destination.IsDoubleStackSlot() && source.IsDoubleStackSlot()) {
5489    Exchange(destination.GetStackIndex(), source.GetStackIndex());
5490    Exchange(destination.GetHighStackIndex(kX86WordSize), source.GetHighStackIndex(kX86WordSize));
5491  } else {
5492    LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination;
5493  }
5494}
5495
5496void ParallelMoveResolverX86::SpillScratch(int reg) {
5497  __ pushl(static_cast<Register>(reg));
5498}
5499
5500void ParallelMoveResolverX86::RestoreScratch(int reg) {
5501  __ popl(static_cast<Register>(reg));
5502}
5503
5504void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
5505  InvokeRuntimeCallingConvention calling_convention;
5506  CodeGenerator::CreateLoadClassLocationSummary(
5507      cls,
5508      Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
5509      Location::RegisterLocation(EAX),
5510      /* code_generator_supports_read_barrier */ true);
5511}
5512
5513void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
5514  LocationSummary* locations = cls->GetLocations();
5515  if (cls->NeedsAccessCheck()) {
5516    codegen_->MoveConstant(locations->GetTemp(0), cls->GetTypeIndex());
5517    codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInitializeTypeAndVerifyAccess),
5518                            cls,
5519                            cls->GetDexPc(),
5520                            nullptr);
5521    CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>();
5522    return;
5523  }
5524
5525  Location out_loc = locations->Out();
5526  Register out = out_loc.AsRegister<Register>();
5527  Register current_method = locations->InAt(0).AsRegister<Register>();
5528
5529  if (cls->IsReferrersClass()) {
5530    DCHECK(!cls->CanCallRuntime());
5531    DCHECK(!cls->MustGenerateClinitCheck());
5532    uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value();
5533    if (kEmitCompilerReadBarrier) {
5534      // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_)
5535      __ leal(out, Address(current_method, declaring_class_offset));
5536      // /* mirror::Class* */ out = out->Read()
5537      codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc);
5538    } else {
5539      // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
5540      __ movl(out, Address(current_method, declaring_class_offset));
5541    }
5542  } else {
5543    // /* GcRoot<mirror::Class>[] */ out =
5544    //        current_method.ptr_sized_fields_->dex_cache_resolved_types_
5545    __ movl(out, Address(current_method,
5546                         ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value()));
5547
5548    size_t cache_offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex());
5549    if (kEmitCompilerReadBarrier) {
5550      // /* GcRoot<mirror::Class>* */ out = &out[type_index]
5551      __ leal(out, Address(out, cache_offset));
5552      // /* mirror::Class* */ out = out->Read()
5553      codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc);
5554    } else {
5555      // /* GcRoot<mirror::Class> */ out = out[type_index]
5556      __ movl(out, Address(out, cache_offset));
5557    }
5558
5559    if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) {
5560      DCHECK(cls->CanCallRuntime());
5561      SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
5562          cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
5563      codegen_->AddSlowPath(slow_path);
5564
5565      if (!cls->IsInDexCache()) {
5566        __ testl(out, out);
5567        __ j(kEqual, slow_path->GetEntryLabel());
5568      }
5569
5570      if (cls->MustGenerateClinitCheck()) {
5571        GenerateClassInitializationCheck(slow_path, out);
5572      } else {
5573        __ Bind(slow_path->GetExitLabel());
5574      }
5575    }
5576  }
5577}
5578
5579void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
5580  LocationSummary* locations =
5581      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
5582  locations->SetInAt(0, Location::RequiresRegister());
5583  if (check->HasUses()) {
5584    locations->SetOut(Location::SameAsFirstInput());
5585  }
5586}
5587
5588void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
5589  // We assume the class to not be null.
5590  SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
5591      check->GetLoadClass(), check, check->GetDexPc(), true);
5592  codegen_->AddSlowPath(slow_path);
5593  GenerateClassInitializationCheck(slow_path,
5594                                   check->GetLocations()->InAt(0).AsRegister<Register>());
5595}
5596
5597void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
5598    SlowPathCode* slow_path, Register class_reg) {
5599  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
5600          Immediate(mirror::Class::kStatusInitialized));
5601  __ j(kLess, slow_path->GetEntryLabel());
5602  __ Bind(slow_path->GetExitLabel());
5603  // No need for memory fence, thanks to the X86 memory model.
5604}
5605
5606void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
5607  LocationSummary* locations =
5608      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
5609  locations->SetInAt(0, Location::RequiresRegister());
5610  locations->SetOut(Location::RequiresRegister());
5611}
5612
5613void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
5614  SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
5615  codegen_->AddSlowPath(slow_path);
5616
5617  LocationSummary* locations = load->GetLocations();
5618  Location out_loc = locations->Out();
5619  Register out = out_loc.AsRegister<Register>();
5620  Register current_method = locations->InAt(0).AsRegister<Register>();
5621
5622  uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value();
5623  if (kEmitCompilerReadBarrier) {
5624    // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_)
5625    __ leal(out, Address(current_method, declaring_class_offset));
5626    // /* mirror::Class* */ out = out->Read()
5627    codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc);
5628  } else {
5629    // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
5630    __ movl(out, Address(current_method, declaring_class_offset));
5631  }
5632
5633  // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
5634  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
5635
5636  size_t cache_offset = CodeGenerator::GetCacheOffset(load->GetStringIndex());
5637  if (kEmitCompilerReadBarrier) {
5638    // /* GcRoot<mirror::String>* */ out = &out[string_index]
5639    __ leal(out, Address(out, cache_offset));
5640    // /* mirror::String* */ out = out->Read()
5641    codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc);
5642  } else {
5643    // /* GcRoot<mirror::String> */ out = out[string_index]
5644    __ movl(out, Address(out, cache_offset));
5645  }
5646
5647  __ testl(out, out);
5648  __ j(kEqual, slow_path->GetEntryLabel());
5649  __ Bind(slow_path->GetExitLabel());
5650}
5651
5652static Address GetExceptionTlsAddress() {
5653  return Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
5654}
5655
5656void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
5657  LocationSummary* locations =
5658      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
5659  locations->SetOut(Location::RequiresRegister());
5660}
5661
5662void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
5663  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), GetExceptionTlsAddress());
5664}
5665
5666void LocationsBuilderX86::VisitClearException(HClearException* clear) {
5667  new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
5668}
5669
5670void InstructionCodeGeneratorX86::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
5671  __ fs()->movl(GetExceptionTlsAddress(), Immediate(0));
5672}
5673
5674void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
5675  LocationSummary* locations =
5676      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
5677  InvokeRuntimeCallingConvention calling_convention;
5678  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
5679}
5680
5681void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
5682  codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pDeliverException),
5683                          instruction,
5684                          instruction->GetDexPc(),
5685                          nullptr);
5686  CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>();
5687}
5688
5689void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
5690  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
5691  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5692  switch (type_check_kind) {
5693    case TypeCheckKind::kExactCheck:
5694    case TypeCheckKind::kAbstractClassCheck:
5695    case TypeCheckKind::kClassHierarchyCheck:
5696    case TypeCheckKind::kArrayObjectCheck:
5697      call_kind =
5698          kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall;
5699      break;
5700    case TypeCheckKind::kArrayCheck:
5701    case TypeCheckKind::kUnresolvedCheck:
5702    case TypeCheckKind::kInterfaceCheck:
5703      call_kind = LocationSummary::kCallOnSlowPath;
5704      break;
5705  }
5706
5707  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5708  locations->SetInAt(0, Location::RequiresRegister());
5709  locations->SetInAt(1, Location::Any());
5710  // Note that TypeCheckSlowPathX86 uses this "out" register too.
5711  locations->SetOut(Location::RequiresRegister());
5712  // When read barriers are enabled, we need a temporary register for
5713  // some cases.
5714  if (kEmitCompilerReadBarrier &&
5715      (type_check_kind == TypeCheckKind::kAbstractClassCheck ||
5716       type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
5717       type_check_kind == TypeCheckKind::kArrayObjectCheck)) {
5718    locations->AddTemp(Location::RequiresRegister());
5719  }
5720}
5721
5722void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
5723  LocationSummary* locations = instruction->GetLocations();
5724  Location obj_loc = locations->InAt(0);
5725  Register obj = obj_loc.AsRegister<Register>();
5726  Location cls = locations->InAt(1);
5727  Location out_loc = locations->Out();
5728  Register out = out_loc.AsRegister<Register>();
5729  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5730  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
5731  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
5732  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
5733  SlowPathCode* slow_path = nullptr;
5734  NearLabel done, zero;
5735
5736  // Return 0 if `obj` is null.
5737  // Avoid null check if we know obj is not null.
5738  if (instruction->MustDoNullCheck()) {
5739    __ testl(obj, obj);
5740    __ j(kEqual, &zero);
5741  }
5742
5743  // /* HeapReference<Class> */ out = obj->klass_
5744  __ movl(out, Address(obj, class_offset));
5745  codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, obj_loc, class_offset);
5746
5747  switch (instruction->GetTypeCheckKind()) {
5748    case TypeCheckKind::kExactCheck: {
5749      if (cls.IsRegister()) {
5750        __ cmpl(out, cls.AsRegister<Register>());
5751      } else {
5752        DCHECK(cls.IsStackSlot()) << cls;
5753        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5754      }
5755
5756      // Classes must be equal for the instanceof to succeed.
5757      __ j(kNotEqual, &zero);
5758      __ movl(out, Immediate(1));
5759      __ jmp(&done);
5760      break;
5761    }
5762
5763    case TypeCheckKind::kAbstractClassCheck: {
5764      // If the class is abstract, we eagerly fetch the super class of the
5765      // object to avoid doing a comparison we know will fail.
5766      NearLabel loop;
5767      __ Bind(&loop);
5768      Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
5769      if (kEmitCompilerReadBarrier) {
5770        // Save the value of `out` into `temp` before overwriting it
5771        // in the following move operation, as we will need it for the
5772        // read barrier below.
5773        Register temp = temp_loc.AsRegister<Register>();
5774        __ movl(temp, out);
5775      }
5776      // /* HeapReference<Class> */ out = out->super_class_
5777      __ movl(out, Address(out, super_offset));
5778      codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset);
5779      __ testl(out, out);
5780      // If `out` is null, we use it for the result, and jump to `done`.
5781      __ j(kEqual, &done);
5782      if (cls.IsRegister()) {
5783        __ cmpl(out, cls.AsRegister<Register>());
5784      } else {
5785        DCHECK(cls.IsStackSlot()) << cls;
5786        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5787      }
5788      __ j(kNotEqual, &loop);
5789      __ movl(out, Immediate(1));
5790      if (zero.IsLinked()) {
5791        __ jmp(&done);
5792      }
5793      break;
5794    }
5795
5796    case TypeCheckKind::kClassHierarchyCheck: {
5797      // Walk over the class hierarchy to find a match.
5798      NearLabel loop, success;
5799      __ Bind(&loop);
5800      if (cls.IsRegister()) {
5801        __ cmpl(out, cls.AsRegister<Register>());
5802      } else {
5803        DCHECK(cls.IsStackSlot()) << cls;
5804        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5805      }
5806      __ j(kEqual, &success);
5807      Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
5808      if (kEmitCompilerReadBarrier) {
5809        // Save the value of `out` into `temp` before overwriting it
5810        // in the following move operation, as we will need it for the
5811        // read barrier below.
5812        Register temp = temp_loc.AsRegister<Register>();
5813        __ movl(temp, out);
5814      }
5815      // /* HeapReference<Class> */ out = out->super_class_
5816      __ movl(out, Address(out, super_offset));
5817      codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset);
5818      __ testl(out, out);
5819      __ j(kNotEqual, &loop);
5820      // If `out` is null, we use it for the result, and jump to `done`.
5821      __ jmp(&done);
5822      __ Bind(&success);
5823      __ movl(out, Immediate(1));
5824      if (zero.IsLinked()) {
5825        __ jmp(&done);
5826      }
5827      break;
5828    }
5829
5830    case TypeCheckKind::kArrayObjectCheck: {
5831      // Do an exact check.
5832      NearLabel exact_check;
5833      if (cls.IsRegister()) {
5834        __ cmpl(out, cls.AsRegister<Register>());
5835      } else {
5836        DCHECK(cls.IsStackSlot()) << cls;
5837        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5838      }
5839      __ j(kEqual, &exact_check);
5840      // Otherwise, we need to check that the object's class is a non-primitive array.
5841      Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
5842      if (kEmitCompilerReadBarrier) {
5843        // Save the value of `out` into `temp` before overwriting it
5844        // in the following move operation, as we will need it for the
5845        // read barrier below.
5846        Register temp = temp_loc.AsRegister<Register>();
5847        __ movl(temp, out);
5848      }
5849      // /* HeapReference<Class> */ out = out->component_type_
5850      __ movl(out, Address(out, component_offset));
5851      codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, component_offset);
5852      __ testl(out, out);
5853      // If `out` is null, we use it for the result, and jump to `done`.
5854      __ j(kEqual, &done);
5855      __ cmpw(Address(out, primitive_offset), Immediate(Primitive::kPrimNot));
5856      __ j(kNotEqual, &zero);
5857      __ Bind(&exact_check);
5858      __ movl(out, Immediate(1));
5859      __ jmp(&done);
5860      break;
5861    }
5862
5863    case TypeCheckKind::kArrayCheck: {
5864      if (cls.IsRegister()) {
5865        __ cmpl(out, cls.AsRegister<Register>());
5866      } else {
5867        DCHECK(cls.IsStackSlot()) << cls;
5868        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5869      }
5870      DCHECK(locations->OnlyCallsOnSlowPath());
5871      slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
5872                                                                    /* is_fatal */ false);
5873      codegen_->AddSlowPath(slow_path);
5874      __ j(kNotEqual, slow_path->GetEntryLabel());
5875      __ movl(out, Immediate(1));
5876      if (zero.IsLinked()) {
5877        __ jmp(&done);
5878      }
5879      break;
5880    }
5881
5882    case TypeCheckKind::kUnresolvedCheck:
5883    case TypeCheckKind::kInterfaceCheck: {
5884      // Note that we indeed only call on slow path, but we always go
5885      // into the slow path for the unresolved & interface check
5886      // cases.
5887      //
5888      // We cannot directly call the InstanceofNonTrivial runtime
5889      // entry point without resorting to a type checking slow path
5890      // here (i.e. by calling InvokeRuntime directly), as it would
5891      // require to assign fixed registers for the inputs of this
5892      // HInstanceOf instruction (following the runtime calling
5893      // convention), which might be cluttered by the potential first
5894      // read barrier emission at the beginning of this method.
5895      DCHECK(locations->OnlyCallsOnSlowPath());
5896      slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
5897                                                                    /* is_fatal */ false);
5898      codegen_->AddSlowPath(slow_path);
5899      __ jmp(slow_path->GetEntryLabel());
5900      if (zero.IsLinked()) {
5901        __ jmp(&done);
5902      }
5903      break;
5904    }
5905  }
5906
5907  if (zero.IsLinked()) {
5908    __ Bind(&zero);
5909    __ xorl(out, out);
5910  }
5911
5912  if (done.IsLinked()) {
5913    __ Bind(&done);
5914  }
5915
5916  if (slow_path != nullptr) {
5917    __ Bind(slow_path->GetExitLabel());
5918  }
5919}
5920
5921void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
5922  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
5923  bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
5924  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5925  switch (type_check_kind) {
5926    case TypeCheckKind::kExactCheck:
5927    case TypeCheckKind::kAbstractClassCheck:
5928    case TypeCheckKind::kClassHierarchyCheck:
5929    case TypeCheckKind::kArrayObjectCheck:
5930      call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ?
5931          LocationSummary::kCallOnSlowPath :
5932          LocationSummary::kNoCall;  // In fact, call on a fatal (non-returning) slow path.
5933      break;
5934    case TypeCheckKind::kArrayCheck:
5935    case TypeCheckKind::kUnresolvedCheck:
5936    case TypeCheckKind::kInterfaceCheck:
5937      call_kind = LocationSummary::kCallOnSlowPath;
5938      break;
5939  }
5940  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5941  locations->SetInAt(0, Location::RequiresRegister());
5942  locations->SetInAt(1, Location::Any());
5943  // Note that TypeCheckSlowPathX86 uses this "temp" register too.
5944  locations->AddTemp(Location::RequiresRegister());
5945  // When read barriers are enabled, we need an additional temporary
5946  // register for some cases.
5947  if (kEmitCompilerReadBarrier &&
5948      (type_check_kind == TypeCheckKind::kAbstractClassCheck ||
5949       type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
5950       type_check_kind == TypeCheckKind::kArrayObjectCheck)) {
5951    locations->AddTemp(Location::RequiresRegister());
5952  }
5953}
5954
5955void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
5956  LocationSummary* locations = instruction->GetLocations();
5957  Location obj_loc = locations->InAt(0);
5958  Register obj = obj_loc.AsRegister<Register>();
5959  Location cls = locations->InAt(1);
5960  Location temp_loc = locations->GetTemp(0);
5961  Register temp = temp_loc.AsRegister<Register>();
5962  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5963  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
5964  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
5965  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
5966
5967  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5968  bool is_type_check_slow_path_fatal =
5969      (type_check_kind == TypeCheckKind::kExactCheck ||
5970       type_check_kind == TypeCheckKind::kAbstractClassCheck ||
5971       type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
5972       type_check_kind == TypeCheckKind::kArrayObjectCheck) &&
5973      !instruction->CanThrowIntoCatchBlock();
5974  SlowPathCode* type_check_slow_path =
5975      new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
5976                                                        is_type_check_slow_path_fatal);
5977  codegen_->AddSlowPath(type_check_slow_path);
5978
5979  NearLabel done;
5980  // Avoid null check if we know obj is not null.
5981  if (instruction->MustDoNullCheck()) {
5982    __ testl(obj, obj);
5983    __ j(kEqual, &done);
5984  }
5985
5986  // /* HeapReference<Class> */ temp = obj->klass_
5987  __ movl(temp, Address(obj, class_offset));
5988  codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
5989
5990  switch (type_check_kind) {
5991    case TypeCheckKind::kExactCheck:
5992    case TypeCheckKind::kArrayCheck: {
5993      if (cls.IsRegister()) {
5994        __ cmpl(temp, cls.AsRegister<Register>());
5995      } else {
5996        DCHECK(cls.IsStackSlot()) << cls;
5997        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
5998      }
5999      // Jump to slow path for throwing the exception or doing a
6000      // more involved array check.
6001      __ j(kNotEqual, type_check_slow_path->GetEntryLabel());
6002      break;
6003    }
6004
6005    case TypeCheckKind::kAbstractClassCheck: {
6006      // If the class is abstract, we eagerly fetch the super class of the
6007      // object to avoid doing a comparison we know will fail.
6008      NearLabel loop, compare_classes;
6009      __ Bind(&loop);
6010      Location temp2_loc =
6011          kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
6012      if (kEmitCompilerReadBarrier) {
6013        // Save the value of `temp` into `temp2` before overwriting it
6014        // in the following move operation, as we will need it for the
6015        // read barrier below.
6016        Register temp2 = temp2_loc.AsRegister<Register>();
6017        __ movl(temp2, temp);
6018      }
6019      // /* HeapReference<Class> */ temp = temp->super_class_
6020      __ movl(temp, Address(temp, super_offset));
6021      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset);
6022
6023      // If the class reference currently in `temp` is not null, jump
6024      // to the `compare_classes` label to compare it with the checked
6025      // class.
6026      __ testl(temp, temp);
6027      __ j(kNotEqual, &compare_classes);
6028      // Otherwise, jump to the slow path to throw the exception.
6029      //
6030      // But before, move back the object's class into `temp` before
6031      // going into the slow path, as it has been overwritten in the
6032      // meantime.
6033      // /* HeapReference<Class> */ temp = obj->klass_
6034      __ movl(temp, Address(obj, class_offset));
6035      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
6036      __ jmp(type_check_slow_path->GetEntryLabel());
6037
6038      __ Bind(&compare_classes);
6039      if (cls.IsRegister()) {
6040        __ cmpl(temp, cls.AsRegister<Register>());
6041      } else {
6042        DCHECK(cls.IsStackSlot()) << cls;
6043        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6044      }
6045      __ j(kNotEqual, &loop);
6046      break;
6047    }
6048
6049    case TypeCheckKind::kClassHierarchyCheck: {
6050      // Walk over the class hierarchy to find a match.
6051      NearLabel loop;
6052      __ Bind(&loop);
6053      if (cls.IsRegister()) {
6054        __ cmpl(temp, cls.AsRegister<Register>());
6055      } else {
6056        DCHECK(cls.IsStackSlot()) << cls;
6057        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6058      }
6059      __ j(kEqual, &done);
6060
6061      Location temp2_loc =
6062          kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
6063      if (kEmitCompilerReadBarrier) {
6064        // Save the value of `temp` into `temp2` before overwriting it
6065        // in the following move operation, as we will need it for the
6066        // read barrier below.
6067        Register temp2 = temp2_loc.AsRegister<Register>();
6068        __ movl(temp2, temp);
6069      }
6070      // /* HeapReference<Class> */ temp = temp->super_class_
6071      __ movl(temp, Address(temp, super_offset));
6072      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset);
6073
6074      // If the class reference currently in `temp` is not null, jump
6075      // back at the beginning of the loop.
6076      __ testl(temp, temp);
6077      __ j(kNotEqual, &loop);
6078      // Otherwise, jump to the slow path to throw the exception.
6079      //
6080      // But before, move back the object's class into `temp` before
6081      // going into the slow path, as it has been overwritten in the
6082      // meantime.
6083      // /* HeapReference<Class> */ temp = obj->klass_
6084      __ movl(temp, Address(obj, class_offset));
6085      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
6086      __ jmp(type_check_slow_path->GetEntryLabel());
6087      break;
6088    }
6089
6090    case TypeCheckKind::kArrayObjectCheck: {
6091      // Do an exact check.
6092      NearLabel check_non_primitive_component_type;
6093      if (cls.IsRegister()) {
6094        __ cmpl(temp, cls.AsRegister<Register>());
6095      } else {
6096        DCHECK(cls.IsStackSlot()) << cls;
6097        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6098      }
6099      __ j(kEqual, &done);
6100
6101      // Otherwise, we need to check that the object's class is a non-primitive array.
6102      Location temp2_loc =
6103          kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
6104      if (kEmitCompilerReadBarrier) {
6105        // Save the value of `temp` into `temp2` before overwriting it
6106        // in the following move operation, as we will need it for the
6107        // read barrier below.
6108        Register temp2 = temp2_loc.AsRegister<Register>();
6109        __ movl(temp2, temp);
6110      }
6111      // /* HeapReference<Class> */ temp = temp->component_type_
6112      __ movl(temp, Address(temp, component_offset));
6113      codegen_->MaybeGenerateReadBarrier(
6114          instruction, temp_loc, temp_loc, temp2_loc, component_offset);
6115
6116      // If the component type is not null (i.e. the object is indeed
6117      // an array), jump to label `check_non_primitive_component_type`
6118      // to further check that this component type is not a primitive
6119      // type.
6120      __ testl(temp, temp);
6121      __ j(kNotEqual, &check_non_primitive_component_type);
6122      // Otherwise, jump to the slow path to throw the exception.
6123      //
6124      // But before, move back the object's class into `temp` before
6125      // going into the slow path, as it has been overwritten in the
6126      // meantime.
6127      // /* HeapReference<Class> */ temp = obj->klass_
6128      __ movl(temp, Address(obj, class_offset));
6129      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
6130      __ jmp(type_check_slow_path->GetEntryLabel());
6131
6132      __ Bind(&check_non_primitive_component_type);
6133      __ cmpw(Address(temp, primitive_offset), Immediate(Primitive::kPrimNot));
6134      __ j(kEqual, &done);
6135      // Same comment as above regarding `temp` and the slow path.
6136      // /* HeapReference<Class> */ temp = obj->klass_
6137      __ movl(temp, Address(obj, class_offset));
6138      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
6139      __ jmp(type_check_slow_path->GetEntryLabel());
6140      break;
6141    }
6142
6143    case TypeCheckKind::kUnresolvedCheck:
6144    case TypeCheckKind::kInterfaceCheck:
6145      // We always go into the type check slow path for the unresolved &
6146      // interface check cases.
6147      //
6148      // We cannot directly call the CheckCast runtime entry point
6149      // without resorting to a type checking slow path here (i.e. by
6150      // calling InvokeRuntime directly), as it would require to
6151      // assign fixed registers for the inputs of this HInstanceOf
6152      // instruction (following the runtime calling convention), which
6153      // might be cluttered by the potential first read barrier
6154      // emission at the beginning of this method.
6155      __ jmp(type_check_slow_path->GetEntryLabel());
6156      break;
6157  }
6158  __ Bind(&done);
6159
6160  __ Bind(type_check_slow_path->GetExitLabel());
6161}
6162
6163void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
6164  LocationSummary* locations =
6165      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
6166  InvokeRuntimeCallingConvention calling_convention;
6167  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
6168}
6169
6170void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
6171  codegen_->InvokeRuntime(instruction->IsEnter() ? QUICK_ENTRY_POINT(pLockObject)
6172                                                 : QUICK_ENTRY_POINT(pUnlockObject),
6173                          instruction,
6174                          instruction->GetDexPc(),
6175                          nullptr);
6176  if (instruction->IsEnter()) {
6177    CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>();
6178  } else {
6179    CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>();
6180  }
6181}
6182
6183void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
6184void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
6185void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
6186
6187void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
6188  LocationSummary* locations =
6189      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
6190  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
6191         || instruction->GetResultType() == Primitive::kPrimLong);
6192  locations->SetInAt(0, Location::RequiresRegister());
6193  locations->SetInAt(1, Location::Any());
6194  locations->SetOut(Location::SameAsFirstInput());
6195}
6196
6197void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
6198  HandleBitwiseOperation(instruction);
6199}
6200
6201void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
6202  HandleBitwiseOperation(instruction);
6203}
6204
6205void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
6206  HandleBitwiseOperation(instruction);
6207}
6208
6209void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
6210  LocationSummary* locations = instruction->GetLocations();
6211  Location first = locations->InAt(0);
6212  Location second = locations->InAt(1);
6213  DCHECK(first.Equals(locations->Out()));
6214
6215  if (instruction->GetResultType() == Primitive::kPrimInt) {
6216    if (second.IsRegister()) {
6217      if (instruction->IsAnd()) {
6218        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
6219      } else if (instruction->IsOr()) {
6220        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
6221      } else {
6222        DCHECK(instruction->IsXor());
6223        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
6224      }
6225    } else if (second.IsConstant()) {
6226      if (instruction->IsAnd()) {
6227        __ andl(first.AsRegister<Register>(),
6228                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6229      } else if (instruction->IsOr()) {
6230        __ orl(first.AsRegister<Register>(),
6231               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6232      } else {
6233        DCHECK(instruction->IsXor());
6234        __ xorl(first.AsRegister<Register>(),
6235                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6236      }
6237    } else {
6238      if (instruction->IsAnd()) {
6239        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6240      } else if (instruction->IsOr()) {
6241        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6242      } else {
6243        DCHECK(instruction->IsXor());
6244        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6245      }
6246    }
6247  } else {
6248    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
6249    if (second.IsRegisterPair()) {
6250      if (instruction->IsAnd()) {
6251        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6252        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6253      } else if (instruction->IsOr()) {
6254        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6255        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6256      } else {
6257        DCHECK(instruction->IsXor());
6258        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6259        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6260      }
6261    } else if (second.IsDoubleStackSlot()) {
6262      if (instruction->IsAnd()) {
6263        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6264        __ andl(first.AsRegisterPairHigh<Register>(),
6265                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6266      } else if (instruction->IsOr()) {
6267        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6268        __ orl(first.AsRegisterPairHigh<Register>(),
6269                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6270      } else {
6271        DCHECK(instruction->IsXor());
6272        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6273        __ xorl(first.AsRegisterPairHigh<Register>(),
6274                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6275      }
6276    } else {
6277      DCHECK(second.IsConstant()) << second;
6278      int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
6279      int32_t low_value = Low32Bits(value);
6280      int32_t high_value = High32Bits(value);
6281      Immediate low(low_value);
6282      Immediate high(high_value);
6283      Register first_low = first.AsRegisterPairLow<Register>();
6284      Register first_high = first.AsRegisterPairHigh<Register>();
6285      if (instruction->IsAnd()) {
6286        if (low_value == 0) {
6287          __ xorl(first_low, first_low);
6288        } else if (low_value != -1) {
6289          __ andl(first_low, low);
6290        }
6291        if (high_value == 0) {
6292          __ xorl(first_high, first_high);
6293        } else if (high_value != -1) {
6294          __ andl(first_high, high);
6295        }
6296      } else if (instruction->IsOr()) {
6297        if (low_value != 0) {
6298          __ orl(first_low, low);
6299        }
6300        if (high_value != 0) {
6301          __ orl(first_high, high);
6302        }
6303      } else {
6304        DCHECK(instruction->IsXor());
6305        if (low_value != 0) {
6306          __ xorl(first_low, low);
6307        }
6308        if (high_value != 0) {
6309          __ xorl(first_high, high);
6310        }
6311      }
6312    }
6313  }
6314}
6315
6316void CodeGeneratorX86::GenerateReadBarrier(HInstruction* instruction,
6317                                           Location out,
6318                                           Location ref,
6319                                           Location obj,
6320                                           uint32_t offset,
6321                                           Location index) {
6322  DCHECK(kEmitCompilerReadBarrier);
6323
6324  // If heap poisoning is enabled, the unpoisoning of the loaded
6325  // reference will be carried out by the runtime within the slow
6326  // path.
6327  //
6328  // Note that `ref` currently does not get unpoisoned (when heap
6329  // poisoning is enabled), which is alright as the `ref` argument is
6330  // not used by the artReadBarrierSlow entry point.
6331  //
6332  // TODO: Unpoison `ref` when it is used by artReadBarrierSlow.
6333  SlowPathCode* slow_path = new (GetGraph()->GetArena())
6334      ReadBarrierForHeapReferenceSlowPathX86(instruction, out, ref, obj, offset, index);
6335  AddSlowPath(slow_path);
6336
6337  // TODO: When read barrier has a fast path, add it here.
6338  /* Currently the read barrier call is inserted after the original load.
6339   * However, if we have a fast path, we need to perform the load of obj.LockWord *before* the
6340   * original load. This load-load ordering is required by the read barrier.
6341   * The fast path/slow path (for Baker's algorithm) should look like:
6342   *
6343   * bool isGray = obj.LockWord & kReadBarrierMask;
6344   * lfence;  // load fence or artificial data dependence to prevent load-load reordering
6345   * ref = obj.field;    // this is the original load
6346   * if (isGray) {
6347   *   ref = Mark(ref);  // ideally the slow path just does Mark(ref)
6348   * }
6349   */
6350
6351  __ jmp(slow_path->GetEntryLabel());
6352  __ Bind(slow_path->GetExitLabel());
6353}
6354
6355void CodeGeneratorX86::MaybeGenerateReadBarrier(HInstruction* instruction,
6356                                                Location out,
6357                                                Location ref,
6358                                                Location obj,
6359                                                uint32_t offset,
6360                                                Location index) {
6361  if (kEmitCompilerReadBarrier) {
6362    // If heap poisoning is enabled, unpoisoning will be taken care of
6363    // by the runtime within the slow path.
6364    GenerateReadBarrier(instruction, out, ref, obj, offset, index);
6365  } else if (kPoisonHeapReferences) {
6366    __ UnpoisonHeapReference(out.AsRegister<Register>());
6367  }
6368}
6369
6370void CodeGeneratorX86::GenerateReadBarrierForRoot(HInstruction* instruction,
6371                                                  Location out,
6372                                                  Location root) {
6373  DCHECK(kEmitCompilerReadBarrier);
6374
6375  // Note that GC roots are not affected by heap poisoning, so we do
6376  // not need to do anything special for this here.
6377  SlowPathCode* slow_path =
6378      new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathX86(instruction, out, root);
6379  AddSlowPath(slow_path);
6380
6381  // TODO: Implement a fast path for ReadBarrierForRoot, performing
6382  // the following operation (for Baker's algorithm):
6383  //
6384  //   if (thread.tls32_.is_gc_marking) {
6385  //     root = Mark(root);
6386  //   }
6387
6388  __ jmp(slow_path->GetEntryLabel());
6389  __ Bind(slow_path->GetExitLabel());
6390}
6391
6392void LocationsBuilderX86::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
6393  // Nothing to do, this should be removed during prepare for register allocator.
6394  LOG(FATAL) << "Unreachable";
6395}
6396
6397void InstructionCodeGeneratorX86::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
6398  // Nothing to do, this should be removed during prepare for register allocator.
6399  LOG(FATAL) << "Unreachable";
6400}
6401
6402void LocationsBuilderX86::VisitFakeString(HFakeString* instruction) {
6403  DCHECK(codegen_->IsBaseline());
6404  LocationSummary* locations =
6405      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
6406  locations->SetOut(Location::ConstantLocation(GetGraph()->GetNullConstant()));
6407}
6408
6409void InstructionCodeGeneratorX86::VisitFakeString(HFakeString* instruction ATTRIBUTE_UNUSED) {
6410  DCHECK(codegen_->IsBaseline());
6411  // Will be generated at use site.
6412}
6413
6414// Simple implementation of packed switch - generate cascaded compare/jumps.
6415void LocationsBuilderX86::VisitPackedSwitch(HPackedSwitch* switch_instr) {
6416  LocationSummary* locations =
6417      new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
6418  locations->SetInAt(0, Location::RequiresRegister());
6419}
6420
6421void InstructionCodeGeneratorX86::VisitPackedSwitch(HPackedSwitch* switch_instr) {
6422  int32_t lower_bound = switch_instr->GetStartValue();
6423  int32_t num_entries = switch_instr->GetNumEntries();
6424  LocationSummary* locations = switch_instr->GetLocations();
6425  Register value_reg = locations->InAt(0).AsRegister<Register>();
6426  HBasicBlock* default_block = switch_instr->GetDefaultBlock();
6427
6428  // Create a series of compare/jumps.
6429  const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors();
6430  for (int i = 0; i < num_entries; i++) {
6431    int32_t case_value = lower_bound + i;
6432    if (case_value == 0) {
6433      __ testl(value_reg, value_reg);
6434    } else {
6435      __ cmpl(value_reg, Immediate(case_value));
6436    }
6437    __ j(kEqual, codegen_->GetLabelOf(successors[i]));
6438  }
6439
6440  // And the default for any other value.
6441  if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) {
6442      __ jmp(codegen_->GetLabelOf(default_block));
6443  }
6444}
6445
6446void LocationsBuilderX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) {
6447  LocationSummary* locations =
6448      new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
6449  locations->SetInAt(0, Location::RequiresRegister());
6450
6451  // Constant area pointer.
6452  locations->SetInAt(1, Location::RequiresRegister());
6453
6454  // And the temporary we need.
6455  locations->AddTemp(Location::RequiresRegister());
6456}
6457
6458void InstructionCodeGeneratorX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) {
6459  int32_t lower_bound = switch_instr->GetStartValue();
6460  int32_t num_entries = switch_instr->GetNumEntries();
6461  LocationSummary* locations = switch_instr->GetLocations();
6462  Register value_reg = locations->InAt(0).AsRegister<Register>();
6463  HBasicBlock* default_block = switch_instr->GetDefaultBlock();
6464
6465  // Optimizing has a jump area.
6466  Register temp_reg = locations->GetTemp(0).AsRegister<Register>();
6467  Register constant_area = locations->InAt(1).AsRegister<Register>();
6468
6469  // Remove the bias, if needed.
6470  if (lower_bound != 0) {
6471    __ leal(temp_reg, Address(value_reg, -lower_bound));
6472    value_reg = temp_reg;
6473  }
6474
6475  // Is the value in range?
6476  DCHECK_GE(num_entries, 1);
6477  __ cmpl(value_reg, Immediate(num_entries - 1));
6478  __ j(kAbove, codegen_->GetLabelOf(default_block));
6479
6480  // We are in the range of the table.
6481  // Load (target-constant_area) from the jump table, indexing by the value.
6482  __ movl(temp_reg, codegen_->LiteralCaseTable(switch_instr, constant_area, value_reg));
6483
6484  // Compute the actual target address by adding in constant_area.
6485  __ addl(temp_reg, constant_area);
6486
6487  // And jump.
6488  __ jmp(temp_reg);
6489}
6490
6491void LocationsBuilderX86::VisitX86ComputeBaseMethodAddress(
6492    HX86ComputeBaseMethodAddress* insn) {
6493  LocationSummary* locations =
6494      new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
6495  locations->SetOut(Location::RequiresRegister());
6496}
6497
6498void InstructionCodeGeneratorX86::VisitX86ComputeBaseMethodAddress(
6499    HX86ComputeBaseMethodAddress* insn) {
6500  LocationSummary* locations = insn->GetLocations();
6501  Register reg = locations->Out().AsRegister<Register>();
6502
6503  // Generate call to next instruction.
6504  Label next_instruction;
6505  __ call(&next_instruction);
6506  __ Bind(&next_instruction);
6507
6508  // Remember this offset for later use with constant area.
6509  codegen_->SetMethodAddressOffset(GetAssembler()->CodeSize());
6510
6511  // Grab the return address off the stack.
6512  __ popl(reg);
6513}
6514
6515void LocationsBuilderX86::VisitX86LoadFromConstantTable(
6516    HX86LoadFromConstantTable* insn) {
6517  LocationSummary* locations =
6518      new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
6519
6520  locations->SetInAt(0, Location::RequiresRegister());
6521  locations->SetInAt(1, Location::ConstantLocation(insn->GetConstant()));
6522
6523  // If we don't need to be materialized, we only need the inputs to be set.
6524  if (!insn->NeedsMaterialization()) {
6525    return;
6526  }
6527
6528  switch (insn->GetType()) {
6529    case Primitive::kPrimFloat:
6530    case Primitive::kPrimDouble:
6531      locations->SetOut(Location::RequiresFpuRegister());
6532      break;
6533
6534    case Primitive::kPrimInt:
6535      locations->SetOut(Location::RequiresRegister());
6536      break;
6537
6538    default:
6539      LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
6540  }
6541}
6542
6543void InstructionCodeGeneratorX86::VisitX86LoadFromConstantTable(HX86LoadFromConstantTable* insn) {
6544  if (!insn->NeedsMaterialization()) {
6545    return;
6546  }
6547
6548  LocationSummary* locations = insn->GetLocations();
6549  Location out = locations->Out();
6550  Register const_area = locations->InAt(0).AsRegister<Register>();
6551  HConstant *value = insn->GetConstant();
6552
6553  switch (insn->GetType()) {
6554    case Primitive::kPrimFloat:
6555      __ movss(out.AsFpuRegister<XmmRegister>(),
6556               codegen_->LiteralFloatAddress(value->AsFloatConstant()->GetValue(), const_area));
6557      break;
6558
6559    case Primitive::kPrimDouble:
6560      __ movsd(out.AsFpuRegister<XmmRegister>(),
6561               codegen_->LiteralDoubleAddress(value->AsDoubleConstant()->GetValue(), const_area));
6562      break;
6563
6564    case Primitive::kPrimInt:
6565      __ movl(out.AsRegister<Register>(),
6566              codegen_->LiteralInt32Address(value->AsIntConstant()->GetValue(), const_area));
6567      break;
6568
6569    default:
6570      LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
6571  }
6572}
6573
6574/**
6575 * Class to handle late fixup of offsets into constant area.
6576 */
6577class RIPFixup : public AssemblerFixup, public ArenaObject<kArenaAllocCodeGenerator> {
6578 public:
6579  RIPFixup(CodeGeneratorX86& codegen, size_t offset)
6580      : codegen_(&codegen), offset_into_constant_area_(offset) {}
6581
6582 protected:
6583  void SetOffset(size_t offset) { offset_into_constant_area_ = offset; }
6584
6585  CodeGeneratorX86* codegen_;
6586
6587 private:
6588  void Process(const MemoryRegion& region, int pos) OVERRIDE {
6589    // Patch the correct offset for the instruction.  The place to patch is the
6590    // last 4 bytes of the instruction.
6591    // The value to patch is the distance from the offset in the constant area
6592    // from the address computed by the HX86ComputeBaseMethodAddress instruction.
6593    int32_t constant_offset = codegen_->ConstantAreaStart() + offset_into_constant_area_;
6594    int32_t relative_position = constant_offset - codegen_->GetMethodAddressOffset();;
6595
6596    // Patch in the right value.
6597    region.StoreUnaligned<int32_t>(pos - 4, relative_position);
6598  }
6599
6600  // Location in constant area that the fixup refers to.
6601  int32_t offset_into_constant_area_;
6602};
6603
6604/**
6605 * Class to handle late fixup of offsets to a jump table that will be created in the
6606 * constant area.
6607 */
6608class JumpTableRIPFixup : public RIPFixup {
6609 public:
6610  JumpTableRIPFixup(CodeGeneratorX86& codegen, HX86PackedSwitch* switch_instr)
6611      : RIPFixup(codegen, static_cast<size_t>(-1)), switch_instr_(switch_instr) {}
6612
6613  void CreateJumpTable() {
6614    X86Assembler* assembler = codegen_->GetAssembler();
6615
6616    // Ensure that the reference to the jump table has the correct offset.
6617    const int32_t offset_in_constant_table = assembler->ConstantAreaSize();
6618    SetOffset(offset_in_constant_table);
6619
6620    // The label values in the jump table are computed relative to the
6621    // instruction addressing the constant area.
6622    const int32_t relative_offset = codegen_->GetMethodAddressOffset();
6623
6624    // Populate the jump table with the correct values for the jump table.
6625    int32_t num_entries = switch_instr_->GetNumEntries();
6626    HBasicBlock* block = switch_instr_->GetBlock();
6627    const ArenaVector<HBasicBlock*>& successors = block->GetSuccessors();
6628    // The value that we want is the target offset - the position of the table.
6629    for (int32_t i = 0; i < num_entries; i++) {
6630      HBasicBlock* b = successors[i];
6631      Label* l = codegen_->GetLabelOf(b);
6632      DCHECK(l->IsBound());
6633      int32_t offset_to_block = l->Position() - relative_offset;
6634      assembler->AppendInt32(offset_to_block);
6635    }
6636  }
6637
6638 private:
6639  const HX86PackedSwitch* switch_instr_;
6640};
6641
6642void CodeGeneratorX86::Finalize(CodeAllocator* allocator) {
6643  // Generate the constant area if needed.
6644  X86Assembler* assembler = GetAssembler();
6645  if (!assembler->IsConstantAreaEmpty() || !fixups_to_jump_tables_.empty()) {
6646    // Align to 4 byte boundary to reduce cache misses, as the data is 4 and 8
6647    // byte values.
6648    assembler->Align(4, 0);
6649    constant_area_start_ = assembler->CodeSize();
6650
6651    // Populate any jump tables.
6652    for (auto jump_table : fixups_to_jump_tables_) {
6653      jump_table->CreateJumpTable();
6654    }
6655
6656    // And now add the constant area to the generated code.
6657    assembler->AddConstantArea();
6658  }
6659
6660  // And finish up.
6661  CodeGenerator::Finalize(allocator);
6662}
6663
6664Address CodeGeneratorX86::LiteralDoubleAddress(double v, Register reg) {
6665  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddDouble(v));
6666  return Address(reg, kDummy32BitOffset, fixup);
6667}
6668
6669Address CodeGeneratorX86::LiteralFloatAddress(float v, Register reg) {
6670  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddFloat(v));
6671  return Address(reg, kDummy32BitOffset, fixup);
6672}
6673
6674Address CodeGeneratorX86::LiteralInt32Address(int32_t v, Register reg) {
6675  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt32(v));
6676  return Address(reg, kDummy32BitOffset, fixup);
6677}
6678
6679Address CodeGeneratorX86::LiteralInt64Address(int64_t v, Register reg) {
6680  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt64(v));
6681  return Address(reg, kDummy32BitOffset, fixup);
6682}
6683
6684Address CodeGeneratorX86::LiteralCaseTable(HX86PackedSwitch* switch_instr,
6685                                           Register reg,
6686                                           Register value) {
6687  // Create a fixup to be used to create and address the jump table.
6688  JumpTableRIPFixup* table_fixup =
6689      new (GetGraph()->GetArena()) JumpTableRIPFixup(*this, switch_instr);
6690
6691  // We have to populate the jump tables.
6692  fixups_to_jump_tables_.push_back(table_fixup);
6693
6694  // We want a scaled address, as we are extracting the correct offset from the table.
6695  return Address(reg, value, TIMES_4, kDummy32BitOffset, table_fixup);
6696}
6697
6698// TODO: target as memory.
6699void CodeGeneratorX86::MoveFromReturnRegister(Location target, Primitive::Type type) {
6700  if (!target.IsValid()) {
6701    DCHECK(type == Primitive::kPrimVoid);
6702    return;
6703  }
6704
6705  DCHECK_NE(type, Primitive::kPrimVoid);
6706
6707  Location return_loc = InvokeDexCallingConventionVisitorX86().GetReturnLocation(type);
6708  if (target.Equals(return_loc)) {
6709    return;
6710  }
6711
6712  // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged
6713  //       with the else branch.
6714  if (type == Primitive::kPrimLong) {
6715    HParallelMove parallel_move(GetGraph()->GetArena());
6716    parallel_move.AddMove(return_loc.ToLow(), target.ToLow(), Primitive::kPrimInt, nullptr);
6717    parallel_move.AddMove(return_loc.ToHigh(), target.ToHigh(), Primitive::kPrimInt, nullptr);
6718    GetMoveResolver()->EmitNativeCode(&parallel_move);
6719  } else {
6720    // Let the parallel move resolver take care of all of this.
6721    HParallelMove parallel_move(GetGraph()->GetArena());
6722    parallel_move.AddMove(return_loc, target, type, nullptr);
6723    GetMoveResolver()->EmitNativeCode(&parallel_move);
6724  }
6725}
6726
6727#undef __
6728
6729}  // namespace x86
6730}  // namespace art
6731