code_generator_x86.cc revision b4536b7de576b20c74c612406c5d3132998075ef 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "art_method.h"
20#include "code_generator_utils.h"
21#include "compiled_method.h"
22#include "entrypoints/quick/quick_entrypoints.h"
23#include "entrypoints/quick/quick_entrypoints_enum.h"
24#include "gc/accounting/card_table.h"
25#include "intrinsics.h"
26#include "intrinsics_x86.h"
27#include "mirror/array-inl.h"
28#include "mirror/class-inl.h"
29#include "pc_relative_fixups_x86.h"
30#include "thread.h"
31#include "utils/assembler.h"
32#include "utils/stack_checks.h"
33#include "utils/x86/assembler_x86.h"
34#include "utils/x86/managed_register_x86.h"
35
36namespace art {
37
38template<class MirrorType>
39class GcRoot;
40
41namespace x86 {
42
43static constexpr int kCurrentMethodStackOffset = 0;
44static constexpr Register kMethodRegisterArgument = EAX;
45
46static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI };
47
48static constexpr int kC2ConditionMask = 0x400;
49
50static constexpr int kFakeReturnRegister = Register(8);
51
52#define __ down_cast<X86Assembler*>(codegen->GetAssembler())->
53#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kX86WordSize, x).Int32Value()
54
55class NullCheckSlowPathX86 : public SlowPathCode {
56 public:
57  explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {}
58
59  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
60    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
61    __ Bind(GetEntryLabel());
62    if (instruction_->CanThrowIntoCatchBlock()) {
63      // Live registers will be restored in the catch block if caught.
64      SaveLiveRegisters(codegen, instruction_->GetLocations());
65    }
66    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowNullPointer),
67                               instruction_,
68                               instruction_->GetDexPc(),
69                               this);
70    CheckEntrypointTypes<kQuickThrowNullPointer, void, void>();
71  }
72
73  bool IsFatal() const OVERRIDE { return true; }
74
75  const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathX86"; }
76
77 private:
78  HNullCheck* const instruction_;
79  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
80};
81
82class DivZeroCheckSlowPathX86 : public SlowPathCode {
83 public:
84  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {}
85
86  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
87    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
88    __ Bind(GetEntryLabel());
89    if (instruction_->CanThrowIntoCatchBlock()) {
90      // Live registers will be restored in the catch block if caught.
91      SaveLiveRegisters(codegen, instruction_->GetLocations());
92    }
93    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowDivZero),
94                               instruction_,
95                               instruction_->GetDexPc(),
96                               this);
97    CheckEntrypointTypes<kQuickThrowDivZero, void, void>();
98  }
99
100  bool IsFatal() const OVERRIDE { return true; }
101
102  const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathX86"; }
103
104 private:
105  HDivZeroCheck* const instruction_;
106  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
107};
108
109class DivRemMinusOneSlowPathX86 : public SlowPathCode {
110 public:
111  DivRemMinusOneSlowPathX86(Register reg, bool is_div) : reg_(reg), is_div_(is_div) {}
112
113  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
114    __ Bind(GetEntryLabel());
115    if (is_div_) {
116      __ negl(reg_);
117    } else {
118      __ movl(reg_, Immediate(0));
119    }
120    __ jmp(GetExitLabel());
121  }
122
123  const char* GetDescription() const OVERRIDE { return "DivRemMinusOneSlowPathX86"; }
124
125 private:
126  Register reg_;
127  bool is_div_;
128  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
129};
130
131class BoundsCheckSlowPathX86 : public SlowPathCode {
132 public:
133  explicit BoundsCheckSlowPathX86(HBoundsCheck* instruction) : instruction_(instruction) {}
134
135  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
136    LocationSummary* locations = instruction_->GetLocations();
137    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
138    __ Bind(GetEntryLabel());
139    // We're moving two locations to locations that could overlap, so we need a parallel
140    // move resolver.
141    if (instruction_->CanThrowIntoCatchBlock()) {
142      // Live registers will be restored in the catch block if caught.
143      SaveLiveRegisters(codegen, instruction_->GetLocations());
144    }
145    InvokeRuntimeCallingConvention calling_convention;
146    x86_codegen->EmitParallelMoves(
147        locations->InAt(0),
148        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
149        Primitive::kPrimInt,
150        locations->InAt(1),
151        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
152        Primitive::kPrimInt);
153    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowArrayBounds),
154                               instruction_,
155                               instruction_->GetDexPc(),
156                               this);
157    CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>();
158  }
159
160  bool IsFatal() const OVERRIDE { return true; }
161
162  const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathX86"; }
163
164 private:
165  HBoundsCheck* const instruction_;
166
167  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
168};
169
170class SuspendCheckSlowPathX86 : public SlowPathCode {
171 public:
172  SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
173      : instruction_(instruction), successor_(successor) {}
174
175  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
176    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
177    __ Bind(GetEntryLabel());
178    SaveLiveRegisters(codegen, instruction_->GetLocations());
179    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pTestSuspend),
180                               instruction_,
181                               instruction_->GetDexPc(),
182                               this);
183    CheckEntrypointTypes<kQuickTestSuspend, void, void>();
184    RestoreLiveRegisters(codegen, instruction_->GetLocations());
185    if (successor_ == nullptr) {
186      __ jmp(GetReturnLabel());
187    } else {
188      __ jmp(x86_codegen->GetLabelOf(successor_));
189    }
190  }
191
192  Label* GetReturnLabel() {
193    DCHECK(successor_ == nullptr);
194    return &return_label_;
195  }
196
197  HBasicBlock* GetSuccessor() const {
198    return successor_;
199  }
200
201  const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathX86"; }
202
203 private:
204  HSuspendCheck* const instruction_;
205  HBasicBlock* const successor_;
206  Label return_label_;
207
208  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
209};
210
211class LoadStringSlowPathX86 : public SlowPathCode {
212 public:
213  explicit LoadStringSlowPathX86(HLoadString* instruction) : instruction_(instruction) {}
214
215  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
216    LocationSummary* locations = instruction_->GetLocations();
217    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
218
219    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
220    __ Bind(GetEntryLabel());
221    SaveLiveRegisters(codegen, locations);
222
223    InvokeRuntimeCallingConvention calling_convention;
224    __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex()));
225    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pResolveString),
226                               instruction_,
227                               instruction_->GetDexPc(),
228                               this);
229    CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
230    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
231    RestoreLiveRegisters(codegen, locations);
232
233    __ jmp(GetExitLabel());
234  }
235
236  const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathX86"; }
237
238 private:
239  HLoadString* const instruction_;
240
241  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
242};
243
244class LoadClassSlowPathX86 : public SlowPathCode {
245 public:
246  LoadClassSlowPathX86(HLoadClass* cls,
247                       HInstruction* at,
248                       uint32_t dex_pc,
249                       bool do_clinit)
250      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
251    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
252  }
253
254  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
255    LocationSummary* locations = at_->GetLocations();
256    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
257    __ Bind(GetEntryLabel());
258    SaveLiveRegisters(codegen, locations);
259
260    InvokeRuntimeCallingConvention calling_convention;
261    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
262    x86_codegen->InvokeRuntime(do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
263                                          : QUICK_ENTRY_POINT(pInitializeType),
264                               at_, dex_pc_, this);
265    if (do_clinit_) {
266      CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>();
267    } else {
268      CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>();
269    }
270
271    // Move the class to the desired location.
272    Location out = locations->Out();
273    if (out.IsValid()) {
274      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
275      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
276    }
277
278    RestoreLiveRegisters(codegen, locations);
279    __ jmp(GetExitLabel());
280  }
281
282  const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathX86"; }
283
284 private:
285  // The class this slow path will load.
286  HLoadClass* const cls_;
287
288  // The instruction where this slow path is happening.
289  // (Might be the load class or an initialization check).
290  HInstruction* const at_;
291
292  // The dex PC of `at_`.
293  const uint32_t dex_pc_;
294
295  // Whether to initialize the class.
296  const bool do_clinit_;
297
298  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
299};
300
301class TypeCheckSlowPathX86 : public SlowPathCode {
302 public:
303  TypeCheckSlowPathX86(HInstruction* instruction, bool is_fatal)
304      : instruction_(instruction), is_fatal_(is_fatal) {}
305
306  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
307    LocationSummary* locations = instruction_->GetLocations();
308    Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0)
309                                                        : locations->Out();
310    DCHECK(instruction_->IsCheckCast()
311           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
312
313    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
314    __ Bind(GetEntryLabel());
315
316    if (!is_fatal_) {
317      SaveLiveRegisters(codegen, locations);
318    }
319
320    // We're moving two locations to locations that could overlap, so we need a parallel
321    // move resolver.
322    InvokeRuntimeCallingConvention calling_convention;
323    x86_codegen->EmitParallelMoves(
324        locations->InAt(1),
325        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
326        Primitive::kPrimNot,
327        object_class,
328        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
329        Primitive::kPrimNot);
330
331    if (instruction_->IsInstanceOf()) {
332      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
333                                 instruction_,
334                                 instruction_->GetDexPc(),
335                                 this);
336      CheckEntrypointTypes<
337          kQuickInstanceofNonTrivial, uint32_t, const mirror::Class*, const mirror::Class*>();
338    } else {
339      DCHECK(instruction_->IsCheckCast());
340      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
341                                 instruction_,
342                                 instruction_->GetDexPc(),
343                                 this);
344      CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
345    }
346
347    if (!is_fatal_) {
348      if (instruction_->IsInstanceOf()) {
349        x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
350      }
351      RestoreLiveRegisters(codegen, locations);
352
353      __ jmp(GetExitLabel());
354    }
355  }
356
357  const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathX86"; }
358  bool IsFatal() const OVERRIDE { return is_fatal_; }
359
360 private:
361  HInstruction* const instruction_;
362  const bool is_fatal_;
363
364  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
365};
366
367class DeoptimizationSlowPathX86 : public SlowPathCode {
368 public:
369  explicit DeoptimizationSlowPathX86(HInstruction* instruction)
370    : instruction_(instruction) {}
371
372  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
373    DCHECK(instruction_->IsDeoptimize());
374    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
375    __ Bind(GetEntryLabel());
376    SaveLiveRegisters(codegen, instruction_->GetLocations());
377    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize),
378                               instruction_,
379                               instruction_->GetDexPc(),
380                               this);
381    CheckEntrypointTypes<kQuickDeoptimize, void, void>();
382  }
383
384  const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathX86"; }
385
386 private:
387  HInstruction* const instruction_;
388  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86);
389};
390
391class ArraySetSlowPathX86 : public SlowPathCode {
392 public:
393  explicit ArraySetSlowPathX86(HInstruction* instruction) : instruction_(instruction) {}
394
395  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
396    LocationSummary* locations = instruction_->GetLocations();
397    __ Bind(GetEntryLabel());
398    SaveLiveRegisters(codegen, locations);
399
400    InvokeRuntimeCallingConvention calling_convention;
401    HParallelMove parallel_move(codegen->GetGraph()->GetArena());
402    parallel_move.AddMove(
403        locations->InAt(0),
404        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
405        Primitive::kPrimNot,
406        nullptr);
407    parallel_move.AddMove(
408        locations->InAt(1),
409        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
410        Primitive::kPrimInt,
411        nullptr);
412    parallel_move.AddMove(
413        locations->InAt(2),
414        Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
415        Primitive::kPrimNot,
416        nullptr);
417    codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
418
419    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
420    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
421                               instruction_,
422                               instruction_->GetDexPc(),
423                               this);
424    CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>();
425    RestoreLiveRegisters(codegen, locations);
426    __ jmp(GetExitLabel());
427  }
428
429  const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathX86"; }
430
431 private:
432  HInstruction* const instruction_;
433
434  DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathX86);
435};
436
437// Slow path generating a read barrier for a heap reference.
438class ReadBarrierForHeapReferenceSlowPathX86 : public SlowPathCode {
439 public:
440  ReadBarrierForHeapReferenceSlowPathX86(HInstruction* instruction,
441                                         Location out,
442                                         Location ref,
443                                         Location obj,
444                                         uint32_t offset,
445                                         Location index)
446      : instruction_(instruction),
447        out_(out),
448        ref_(ref),
449        obj_(obj),
450        offset_(offset),
451        index_(index) {
452    DCHECK(kEmitCompilerReadBarrier);
453    // If `obj` is equal to `out` or `ref`, it means the initial object
454    // has been overwritten by (or after) the heap object reference load
455    // to be instrumented, e.g.:
456    //
457    //   __ movl(out, Address(out, offset));
458    //   codegen_->GenerateReadBarrier(instruction, out_loc, out_loc, out_loc, offset);
459    //
460    // In that case, we have lost the information about the original
461    // object, and the emitted read barrier cannot work properly.
462    DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out;
463    DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref;
464  }
465
466  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
467    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
468    LocationSummary* locations = instruction_->GetLocations();
469    Register reg_out = out_.AsRegister<Register>();
470    DCHECK(locations->CanCall());
471    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
472    DCHECK(!instruction_->IsInvoke() ||
473           (instruction_->IsInvokeStaticOrDirect() &&
474            instruction_->GetLocations()->Intrinsified()));
475
476    __ Bind(GetEntryLabel());
477    SaveLiveRegisters(codegen, locations);
478
479    // We may have to change the index's value, but as `index_` is a
480    // constant member (like other "inputs" of this slow path),
481    // introduce a copy of it, `index`.
482    Location index = index_;
483    if (index_.IsValid()) {
484      // Handle `index_` for HArrayGet and intrinsic UnsafeGetObject.
485      if (instruction_->IsArrayGet()) {
486        // Compute the actual memory offset and store it in `index`.
487        Register index_reg = index_.AsRegister<Register>();
488        DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg));
489        if (codegen->IsCoreCalleeSaveRegister(index_reg)) {
490          // We are about to change the value of `index_reg` (see the
491          // calls to art::x86::X86Assembler::shll and
492          // art::x86::X86Assembler::AddImmediate below), but it has
493          // not been saved by the previous call to
494          // art::SlowPathCode::SaveLiveRegisters, as it is a
495          // callee-save register --
496          // art::SlowPathCode::SaveLiveRegisters does not consider
497          // callee-save registers, as it has been designed with the
498          // assumption that callee-save registers are supposed to be
499          // handled by the called function.  So, as a callee-save
500          // register, `index_reg` _would_ eventually be saved onto
501          // the stack, but it would be too late: we would have
502          // changed its value earlier.  Therefore, we manually save
503          // it here into another freely available register,
504          // `free_reg`, chosen of course among the caller-save
505          // registers (as a callee-save `free_reg` register would
506          // exhibit the same problem).
507          //
508          // Note we could have requested a temporary register from
509          // the register allocator instead; but we prefer not to, as
510          // this is a slow path, and we know we can find a
511          // caller-save register that is available.
512          Register free_reg = FindAvailableCallerSaveRegister(codegen);
513          __ movl(free_reg, index_reg);
514          index_reg = free_reg;
515          index = Location::RegisterLocation(index_reg);
516        } else {
517          // The initial register stored in `index_` has already been
518          // saved in the call to art::SlowPathCode::SaveLiveRegisters
519          // (as it is not a callee-save register), so we can freely
520          // use it.
521        }
522        // Shifting the index value contained in `index_reg` by the scale
523        // factor (2) cannot overflow in practice, as the runtime is
524        // unable to allocate object arrays with a size larger than
525        // 2^26 - 1 (that is, 2^28 - 4 bytes).
526        __ shll(index_reg, Immediate(TIMES_4));
527        static_assert(
528            sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
529            "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
530        __ AddImmediate(index_reg, Immediate(offset_));
531      } else {
532        DCHECK(instruction_->IsInvoke());
533        DCHECK(instruction_->GetLocations()->Intrinsified());
534        DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) ||
535               (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile))
536            << instruction_->AsInvoke()->GetIntrinsic();
537        DCHECK_EQ(offset_, 0U);
538        DCHECK(index_.IsRegisterPair());
539        // UnsafeGet's offset location is a register pair, the low
540        // part contains the correct offset.
541        index = index_.ToLow();
542      }
543    }
544
545    // We're moving two or three locations to locations that could
546    // overlap, so we need a parallel move resolver.
547    InvokeRuntimeCallingConvention calling_convention;
548    HParallelMove parallel_move(codegen->GetGraph()->GetArena());
549    parallel_move.AddMove(ref_,
550                          Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
551                          Primitive::kPrimNot,
552                          nullptr);
553    parallel_move.AddMove(obj_,
554                          Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
555                          Primitive::kPrimNot,
556                          nullptr);
557    if (index.IsValid()) {
558      parallel_move.AddMove(index,
559                            Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
560                            Primitive::kPrimInt,
561                            nullptr);
562      codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
563    } else {
564      codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
565      __ movl(calling_convention.GetRegisterAt(2), Immediate(offset_));
566    }
567    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierSlow),
568                               instruction_,
569                               instruction_->GetDexPc(),
570                               this);
571    CheckEntrypointTypes<
572        kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>();
573    x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
574
575    RestoreLiveRegisters(codegen, locations);
576    __ jmp(GetExitLabel());
577  }
578
579  const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathX86"; }
580
581 private:
582  Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) {
583    size_t ref = static_cast<int>(ref_.AsRegister<Register>());
584    size_t obj = static_cast<int>(obj_.AsRegister<Register>());
585    for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
586      if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) {
587        return static_cast<Register>(i);
588      }
589    }
590    // We shall never fail to find a free caller-save register, as
591    // there are more than two core caller-save registers on x86
592    // (meaning it is possible to find one which is different from
593    // `ref` and `obj`).
594    DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u);
595    LOG(FATAL) << "Could not find a free caller-save register";
596    UNREACHABLE();
597  }
598
599  HInstruction* const instruction_;
600  const Location out_;
601  const Location ref_;
602  const Location obj_;
603  const uint32_t offset_;
604  // An additional location containing an index to an array.
605  // Only used for HArrayGet and the UnsafeGetObject &
606  // UnsafeGetObjectVolatile intrinsics.
607  const Location index_;
608
609  DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathX86);
610};
611
612// Slow path generating a read barrier for a GC root.
613class ReadBarrierForRootSlowPathX86 : public SlowPathCode {
614 public:
615  ReadBarrierForRootSlowPathX86(HInstruction* instruction, Location out, Location root)
616      : instruction_(instruction), out_(out), root_(root) {}
617
618  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
619    LocationSummary* locations = instruction_->GetLocations();
620    Register reg_out = out_.AsRegister<Register>();
621    DCHECK(locations->CanCall());
622    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
623    DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString());
624
625    __ Bind(GetEntryLabel());
626    SaveLiveRegisters(codegen, locations);
627
628    InvokeRuntimeCallingConvention calling_convention;
629    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
630    x86_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_);
631    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierForRootSlow),
632                               instruction_,
633                               instruction_->GetDexPc(),
634                               this);
635    CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>();
636    x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
637
638    RestoreLiveRegisters(codegen, locations);
639    __ jmp(GetExitLabel());
640  }
641
642  const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathX86"; }
643
644 private:
645  HInstruction* const instruction_;
646  const Location out_;
647  const Location root_;
648
649  DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathX86);
650};
651
652#undef __
653#define __ down_cast<X86Assembler*>(GetAssembler())->
654
655inline Condition X86Condition(IfCondition cond) {
656  switch (cond) {
657    case kCondEQ: return kEqual;
658    case kCondNE: return kNotEqual;
659    case kCondLT: return kLess;
660    case kCondLE: return kLessEqual;
661    case kCondGT: return kGreater;
662    case kCondGE: return kGreaterEqual;
663    case kCondB:  return kBelow;
664    case kCondBE: return kBelowEqual;
665    case kCondA:  return kAbove;
666    case kCondAE: return kAboveEqual;
667  }
668  LOG(FATAL) << "Unreachable";
669  UNREACHABLE();
670}
671
672// Maps signed condition to unsigned condition and FP condition to x86 name.
673inline Condition X86UnsignedOrFPCondition(IfCondition cond) {
674  switch (cond) {
675    case kCondEQ: return kEqual;
676    case kCondNE: return kNotEqual;
677    // Signed to unsigned, and FP to x86 name.
678    case kCondLT: return kBelow;
679    case kCondLE: return kBelowEqual;
680    case kCondGT: return kAbove;
681    case kCondGE: return kAboveEqual;
682    // Unsigned remain unchanged.
683    case kCondB:  return kBelow;
684    case kCondBE: return kBelowEqual;
685    case kCondA:  return kAbove;
686    case kCondAE: return kAboveEqual;
687  }
688  LOG(FATAL) << "Unreachable";
689  UNREACHABLE();
690}
691
692void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
693  stream << Register(reg);
694}
695
696void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
697  stream << XmmRegister(reg);
698}
699
700size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
701  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
702  return kX86WordSize;
703}
704
705size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
706  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
707  return kX86WordSize;
708}
709
710size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
711  __ movsd(Address(ESP, stack_index), XmmRegister(reg_id));
712  return GetFloatingPointSpillSlotSize();
713}
714
715size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
716  __ movsd(XmmRegister(reg_id), Address(ESP, stack_index));
717  return GetFloatingPointSpillSlotSize();
718}
719
720void CodeGeneratorX86::InvokeRuntime(QuickEntrypointEnum entrypoint,
721                                     HInstruction* instruction,
722                                     uint32_t dex_pc,
723                                     SlowPathCode* slow_path) {
724  InvokeRuntime(GetThreadOffset<kX86WordSize>(entrypoint).Int32Value(),
725                instruction,
726                dex_pc,
727                slow_path);
728}
729
730void CodeGeneratorX86::InvokeRuntime(int32_t entry_point_offset,
731                                     HInstruction* instruction,
732                                     uint32_t dex_pc,
733                                     SlowPathCode* slow_path) {
734  ValidateInvokeRuntime(instruction, slow_path);
735  __ fs()->call(Address::Absolute(entry_point_offset));
736  RecordPcInfo(instruction, dex_pc, slow_path);
737}
738
739CodeGeneratorX86::CodeGeneratorX86(HGraph* graph,
740                                   const X86InstructionSetFeatures& isa_features,
741                                   const CompilerOptions& compiler_options,
742                                   OptimizingCompilerStats* stats)
743    : CodeGenerator(graph,
744                    kNumberOfCpuRegisters,
745                    kNumberOfXmmRegisters,
746                    kNumberOfRegisterPairs,
747                    ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
748                                        arraysize(kCoreCalleeSaves))
749                        | (1 << kFakeReturnRegister),
750                    0,
751                    compiler_options,
752                    stats),
753      block_labels_(nullptr),
754      location_builder_(graph, this),
755      instruction_visitor_(graph, this),
756      move_resolver_(graph->GetArena(), this),
757      isa_features_(isa_features),
758      method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
759      relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
760      pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
761      fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
762  // Use a fake return address register to mimic Quick.
763  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
764}
765
766Location CodeGeneratorX86::AllocateFreeRegister(Primitive::Type type) const {
767  switch (type) {
768    case Primitive::kPrimLong: {
769      size_t reg = FindFreeEntry(blocked_register_pairs_, kNumberOfRegisterPairs);
770      X86ManagedRegister pair =
771          X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(reg));
772      DCHECK(!blocked_core_registers_[pair.AsRegisterPairLow()]);
773      DCHECK(!blocked_core_registers_[pair.AsRegisterPairHigh()]);
774      blocked_core_registers_[pair.AsRegisterPairLow()] = true;
775      blocked_core_registers_[pair.AsRegisterPairHigh()] = true;
776      UpdateBlockedPairRegisters();
777      return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
778    }
779
780    case Primitive::kPrimByte:
781    case Primitive::kPrimBoolean:
782    case Primitive::kPrimChar:
783    case Primitive::kPrimShort:
784    case Primitive::kPrimInt:
785    case Primitive::kPrimNot: {
786      Register reg = static_cast<Register>(
787          FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters));
788      // Block all register pairs that contain `reg`.
789      for (int i = 0; i < kNumberOfRegisterPairs; i++) {
790        X86ManagedRegister current =
791            X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
792        if (current.AsRegisterPairLow() == reg || current.AsRegisterPairHigh() == reg) {
793          blocked_register_pairs_[i] = true;
794        }
795      }
796      return Location::RegisterLocation(reg);
797    }
798
799    case Primitive::kPrimFloat:
800    case Primitive::kPrimDouble: {
801      return Location::FpuRegisterLocation(
802          FindFreeEntry(blocked_fpu_registers_, kNumberOfXmmRegisters));
803    }
804
805    case Primitive::kPrimVoid:
806      LOG(FATAL) << "Unreachable type " << type;
807  }
808
809  return Location::NoLocation();
810}
811
812void CodeGeneratorX86::SetupBlockedRegisters(bool is_baseline) const {
813  // Don't allocate the dalvik style register pair passing.
814  blocked_register_pairs_[ECX_EDX] = true;
815
816  // Stack register is always reserved.
817  blocked_core_registers_[ESP] = true;
818
819  if (is_baseline) {
820    blocked_core_registers_[EBP] = true;
821    blocked_core_registers_[ESI] = true;
822    blocked_core_registers_[EDI] = true;
823  }
824
825  UpdateBlockedPairRegisters();
826}
827
828void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
829  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
830    X86ManagedRegister current =
831        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
832    if (blocked_core_registers_[current.AsRegisterPairLow()]
833        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
834      blocked_register_pairs_[i] = true;
835    }
836  }
837}
838
839InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
840      : HGraphVisitor(graph),
841        assembler_(codegen->GetAssembler()),
842        codegen_(codegen) {}
843
844static dwarf::Reg DWARFReg(Register reg) {
845  return dwarf::Reg::X86Core(static_cast<int>(reg));
846}
847
848void CodeGeneratorX86::GenerateFrameEntry() {
849  __ cfi().SetCurrentCFAOffset(kX86WordSize);  // return address
850  __ Bind(&frame_entry_label_);
851  bool skip_overflow_check =
852      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
853  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
854
855  if (!skip_overflow_check) {
856    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
857    RecordPcInfo(nullptr, 0);
858  }
859
860  if (HasEmptyFrame()) {
861    return;
862  }
863
864  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
865    Register reg = kCoreCalleeSaves[i];
866    if (allocated_registers_.ContainsCoreRegister(reg)) {
867      __ pushl(reg);
868      __ cfi().AdjustCFAOffset(kX86WordSize);
869      __ cfi().RelOffset(DWARFReg(reg), 0);
870    }
871  }
872
873  int adjust = GetFrameSize() - FrameEntrySpillSize();
874  __ subl(ESP, Immediate(adjust));
875  __ cfi().AdjustCFAOffset(adjust);
876  __ movl(Address(ESP, kCurrentMethodStackOffset), kMethodRegisterArgument);
877}
878
879void CodeGeneratorX86::GenerateFrameExit() {
880  __ cfi().RememberState();
881  if (!HasEmptyFrame()) {
882    int adjust = GetFrameSize() - FrameEntrySpillSize();
883    __ addl(ESP, Immediate(adjust));
884    __ cfi().AdjustCFAOffset(-adjust);
885
886    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
887      Register reg = kCoreCalleeSaves[i];
888      if (allocated_registers_.ContainsCoreRegister(reg)) {
889        __ popl(reg);
890        __ cfi().AdjustCFAOffset(-static_cast<int>(kX86WordSize));
891        __ cfi().Restore(DWARFReg(reg));
892      }
893    }
894  }
895  __ ret();
896  __ cfi().RestoreState();
897  __ cfi().DefCFAOffset(GetFrameSize());
898}
899
900void CodeGeneratorX86::Bind(HBasicBlock* block) {
901  __ Bind(GetLabelOf(block));
902}
903
904Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
905  switch (load->GetType()) {
906    case Primitive::kPrimLong:
907    case Primitive::kPrimDouble:
908      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
909
910    case Primitive::kPrimInt:
911    case Primitive::kPrimNot:
912    case Primitive::kPrimFloat:
913      return Location::StackSlot(GetStackSlot(load->GetLocal()));
914
915    case Primitive::kPrimBoolean:
916    case Primitive::kPrimByte:
917    case Primitive::kPrimChar:
918    case Primitive::kPrimShort:
919    case Primitive::kPrimVoid:
920      LOG(FATAL) << "Unexpected type " << load->GetType();
921      UNREACHABLE();
922  }
923
924  LOG(FATAL) << "Unreachable";
925  UNREACHABLE();
926}
927
928Location InvokeDexCallingConventionVisitorX86::GetReturnLocation(Primitive::Type type) const {
929  switch (type) {
930    case Primitive::kPrimBoolean:
931    case Primitive::kPrimByte:
932    case Primitive::kPrimChar:
933    case Primitive::kPrimShort:
934    case Primitive::kPrimInt:
935    case Primitive::kPrimNot:
936      return Location::RegisterLocation(EAX);
937
938    case Primitive::kPrimLong:
939      return Location::RegisterPairLocation(EAX, EDX);
940
941    case Primitive::kPrimVoid:
942      return Location::NoLocation();
943
944    case Primitive::kPrimDouble:
945    case Primitive::kPrimFloat:
946      return Location::FpuRegisterLocation(XMM0);
947  }
948
949  UNREACHABLE();
950}
951
952Location InvokeDexCallingConventionVisitorX86::GetMethodLocation() const {
953  return Location::RegisterLocation(kMethodRegisterArgument);
954}
955
956Location InvokeDexCallingConventionVisitorX86::GetNextLocation(Primitive::Type type) {
957  switch (type) {
958    case Primitive::kPrimBoolean:
959    case Primitive::kPrimByte:
960    case Primitive::kPrimChar:
961    case Primitive::kPrimShort:
962    case Primitive::kPrimInt:
963    case Primitive::kPrimNot: {
964      uint32_t index = gp_index_++;
965      stack_index_++;
966      if (index < calling_convention.GetNumberOfRegisters()) {
967        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
968      } else {
969        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
970      }
971    }
972
973    case Primitive::kPrimLong: {
974      uint32_t index = gp_index_;
975      gp_index_ += 2;
976      stack_index_ += 2;
977      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
978        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
979            calling_convention.GetRegisterPairAt(index));
980        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
981      } else {
982        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
983      }
984    }
985
986    case Primitive::kPrimFloat: {
987      uint32_t index = float_index_++;
988      stack_index_++;
989      if (index < calling_convention.GetNumberOfFpuRegisters()) {
990        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
991      } else {
992        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
993      }
994    }
995
996    case Primitive::kPrimDouble: {
997      uint32_t index = float_index_++;
998      stack_index_ += 2;
999      if (index < calling_convention.GetNumberOfFpuRegisters()) {
1000        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
1001      } else {
1002        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
1003      }
1004    }
1005
1006    case Primitive::kPrimVoid:
1007      LOG(FATAL) << "Unexpected parameter type " << type;
1008      break;
1009  }
1010  return Location::NoLocation();
1011}
1012
1013void CodeGeneratorX86::Move32(Location destination, Location source) {
1014  if (source.Equals(destination)) {
1015    return;
1016  }
1017  if (destination.IsRegister()) {
1018    if (source.IsRegister()) {
1019      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
1020    } else if (source.IsFpuRegister()) {
1021      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
1022    } else {
1023      DCHECK(source.IsStackSlot());
1024      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
1025    }
1026  } else if (destination.IsFpuRegister()) {
1027    if (source.IsRegister()) {
1028      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
1029    } else if (source.IsFpuRegister()) {
1030      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
1031    } else {
1032      DCHECK(source.IsStackSlot());
1033      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
1034    }
1035  } else {
1036    DCHECK(destination.IsStackSlot()) << destination;
1037    if (source.IsRegister()) {
1038      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
1039    } else if (source.IsFpuRegister()) {
1040      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
1041    } else if (source.IsConstant()) {
1042      HConstant* constant = source.GetConstant();
1043      int32_t value = GetInt32ValueOf(constant);
1044      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
1045    } else {
1046      DCHECK(source.IsStackSlot());
1047      __ pushl(Address(ESP, source.GetStackIndex()));
1048      __ popl(Address(ESP, destination.GetStackIndex()));
1049    }
1050  }
1051}
1052
1053void CodeGeneratorX86::Move64(Location destination, Location source) {
1054  if (source.Equals(destination)) {
1055    return;
1056  }
1057  if (destination.IsRegisterPair()) {
1058    if (source.IsRegisterPair()) {
1059      EmitParallelMoves(
1060          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
1061          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
1062          Primitive::kPrimInt,
1063          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
1064          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
1065          Primitive::kPrimInt);
1066    } else if (source.IsFpuRegister()) {
1067      XmmRegister src_reg = source.AsFpuRegister<XmmRegister>();
1068      __ movd(destination.AsRegisterPairLow<Register>(), src_reg);
1069      __ psrlq(src_reg, Immediate(32));
1070      __ movd(destination.AsRegisterPairHigh<Register>(), src_reg);
1071    } else {
1072      // No conflict possible, so just do the moves.
1073      DCHECK(source.IsDoubleStackSlot());
1074      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
1075      __ movl(destination.AsRegisterPairHigh<Register>(),
1076              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
1077    }
1078  } else if (destination.IsFpuRegister()) {
1079    if (source.IsFpuRegister()) {
1080      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
1081    } else if (source.IsDoubleStackSlot()) {
1082      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
1083    } else if (source.IsRegisterPair()) {
1084      size_t elem_size = Primitive::ComponentSize(Primitive::kPrimInt);
1085      // Create stack space for 2 elements.
1086      __ subl(ESP, Immediate(2 * elem_size));
1087      __ movl(Address(ESP, 0), source.AsRegisterPairLow<Register>());
1088      __ movl(Address(ESP, elem_size), source.AsRegisterPairHigh<Register>());
1089      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
1090      // And remove the temporary stack space we allocated.
1091      __ addl(ESP, Immediate(2 * elem_size));
1092    } else {
1093      LOG(FATAL) << "Unimplemented";
1094    }
1095  } else {
1096    DCHECK(destination.IsDoubleStackSlot()) << destination;
1097    if (source.IsRegisterPair()) {
1098      // No conflict possible, so just do the moves.
1099      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
1100      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
1101              source.AsRegisterPairHigh<Register>());
1102    } else if (source.IsFpuRegister()) {
1103      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
1104    } else if (source.IsConstant()) {
1105      HConstant* constant = source.GetConstant();
1106      int64_t value;
1107      if (constant->IsLongConstant()) {
1108        value = constant->AsLongConstant()->GetValue();
1109      } else {
1110        DCHECK(constant->IsDoubleConstant());
1111        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
1112      }
1113      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(Low32Bits(value)));
1114      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value)));
1115    } else {
1116      DCHECK(source.IsDoubleStackSlot()) << source;
1117      EmitParallelMoves(
1118          Location::StackSlot(source.GetStackIndex()),
1119          Location::StackSlot(destination.GetStackIndex()),
1120          Primitive::kPrimInt,
1121          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
1122          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)),
1123          Primitive::kPrimInt);
1124    }
1125  }
1126}
1127
1128void CodeGeneratorX86::Move(HInstruction* instruction, Location location, HInstruction* move_for) {
1129  LocationSummary* locations = instruction->GetLocations();
1130  if (instruction->IsCurrentMethod()) {
1131    Move32(location, Location::StackSlot(kCurrentMethodStackOffset));
1132  } else if (locations != nullptr && locations->Out().Equals(location)) {
1133    return;
1134  } else if (locations != nullptr && locations->Out().IsConstant()) {
1135    HConstant* const_to_move = locations->Out().GetConstant();
1136    if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
1137      Immediate imm(GetInt32ValueOf(const_to_move));
1138      if (location.IsRegister()) {
1139        __ movl(location.AsRegister<Register>(), imm);
1140      } else if (location.IsStackSlot()) {
1141        __ movl(Address(ESP, location.GetStackIndex()), imm);
1142      } else {
1143        DCHECK(location.IsConstant());
1144        DCHECK_EQ(location.GetConstant(), const_to_move);
1145      }
1146    } else if (const_to_move->IsLongConstant()) {
1147      int64_t value = const_to_move->AsLongConstant()->GetValue();
1148      if (location.IsRegisterPair()) {
1149        __ movl(location.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
1150        __ movl(location.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
1151      } else if (location.IsDoubleStackSlot()) {
1152        __ movl(Address(ESP, location.GetStackIndex()), Immediate(Low32Bits(value)));
1153        __ movl(Address(ESP, location.GetHighStackIndex(kX86WordSize)),
1154                Immediate(High32Bits(value)));
1155      } else {
1156        DCHECK(location.IsConstant());
1157        DCHECK_EQ(location.GetConstant(), instruction);
1158      }
1159    }
1160  } else if (instruction->IsTemporary()) {
1161    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
1162    if (temp_location.IsStackSlot()) {
1163      Move32(location, temp_location);
1164    } else {
1165      DCHECK(temp_location.IsDoubleStackSlot());
1166      Move64(location, temp_location);
1167    }
1168  } else if (instruction->IsLoadLocal()) {
1169    int slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
1170    switch (instruction->GetType()) {
1171      case Primitive::kPrimBoolean:
1172      case Primitive::kPrimByte:
1173      case Primitive::kPrimChar:
1174      case Primitive::kPrimShort:
1175      case Primitive::kPrimInt:
1176      case Primitive::kPrimNot:
1177      case Primitive::kPrimFloat:
1178        Move32(location, Location::StackSlot(slot));
1179        break;
1180
1181      case Primitive::kPrimLong:
1182      case Primitive::kPrimDouble:
1183        Move64(location, Location::DoubleStackSlot(slot));
1184        break;
1185
1186      default:
1187        LOG(FATAL) << "Unimplemented local type " << instruction->GetType();
1188    }
1189  } else {
1190    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
1191    switch (instruction->GetType()) {
1192      case Primitive::kPrimBoolean:
1193      case Primitive::kPrimByte:
1194      case Primitive::kPrimChar:
1195      case Primitive::kPrimShort:
1196      case Primitive::kPrimInt:
1197      case Primitive::kPrimNot:
1198      case Primitive::kPrimFloat:
1199        Move32(location, locations->Out());
1200        break;
1201
1202      case Primitive::kPrimLong:
1203      case Primitive::kPrimDouble:
1204        Move64(location, locations->Out());
1205        break;
1206
1207      default:
1208        LOG(FATAL) << "Unexpected type " << instruction->GetType();
1209    }
1210  }
1211}
1212
1213void CodeGeneratorX86::MoveConstant(Location location, int32_t value) {
1214  DCHECK(location.IsRegister());
1215  __ movl(location.AsRegister<Register>(), Immediate(value));
1216}
1217
1218void CodeGeneratorX86::MoveLocation(Location dst, Location src, Primitive::Type dst_type) {
1219  if (Primitive::Is64BitType(dst_type)) {
1220    Move64(dst, src);
1221  } else {
1222    Move32(dst, src);
1223  }
1224}
1225
1226void CodeGeneratorX86::AddLocationAsTemp(Location location, LocationSummary* locations) {
1227  if (location.IsRegister()) {
1228    locations->AddTemp(location);
1229  } else if (location.IsRegisterPair()) {
1230    locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>()));
1231    locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>()));
1232  } else {
1233    UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location;
1234  }
1235}
1236
1237void InstructionCodeGeneratorX86::HandleGoto(HInstruction* got, HBasicBlock* successor) {
1238  DCHECK(!successor->IsExitBlock());
1239
1240  HBasicBlock* block = got->GetBlock();
1241  HInstruction* previous = got->GetPrevious();
1242
1243  HLoopInformation* info = block->GetLoopInformation();
1244  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
1245    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
1246    return;
1247  }
1248
1249  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
1250    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
1251  }
1252  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
1253    __ jmp(codegen_->GetLabelOf(successor));
1254  }
1255}
1256
1257void LocationsBuilderX86::VisitGoto(HGoto* got) {
1258  got->SetLocations(nullptr);
1259}
1260
1261void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
1262  HandleGoto(got, got->GetSuccessor());
1263}
1264
1265void LocationsBuilderX86::VisitTryBoundary(HTryBoundary* try_boundary) {
1266  try_boundary->SetLocations(nullptr);
1267}
1268
1269void InstructionCodeGeneratorX86::VisitTryBoundary(HTryBoundary* try_boundary) {
1270  HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
1271  if (!successor->IsExitBlock()) {
1272    HandleGoto(try_boundary, successor);
1273  }
1274}
1275
1276void LocationsBuilderX86::VisitExit(HExit* exit) {
1277  exit->SetLocations(nullptr);
1278}
1279
1280void InstructionCodeGeneratorX86::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
1281}
1282
1283void InstructionCodeGeneratorX86::GenerateFPJumps(HCondition* cond,
1284                                                  Label* true_label,
1285                                                  Label* false_label) {
1286  if (cond->IsFPConditionTrueIfNaN()) {
1287    __ j(kUnordered, true_label);
1288  } else if (cond->IsFPConditionFalseIfNaN()) {
1289    __ j(kUnordered, false_label);
1290  }
1291  __ j(X86UnsignedOrFPCondition(cond->GetCondition()), true_label);
1292}
1293
1294void InstructionCodeGeneratorX86::GenerateLongComparesAndJumps(HCondition* cond,
1295                                                               Label* true_label,
1296                                                               Label* false_label) {
1297  LocationSummary* locations = cond->GetLocations();
1298  Location left = locations->InAt(0);
1299  Location right = locations->InAt(1);
1300  IfCondition if_cond = cond->GetCondition();
1301
1302  Register left_high = left.AsRegisterPairHigh<Register>();
1303  Register left_low = left.AsRegisterPairLow<Register>();
1304  IfCondition true_high_cond = if_cond;
1305  IfCondition false_high_cond = cond->GetOppositeCondition();
1306  Condition final_condition = X86UnsignedOrFPCondition(if_cond);  // unsigned on lower part
1307
1308  // Set the conditions for the test, remembering that == needs to be
1309  // decided using the low words.
1310  switch (if_cond) {
1311    case kCondEQ:
1312    case kCondNE:
1313      // Nothing to do.
1314      break;
1315    case kCondLT:
1316      false_high_cond = kCondGT;
1317      break;
1318    case kCondLE:
1319      true_high_cond = kCondLT;
1320      break;
1321    case kCondGT:
1322      false_high_cond = kCondLT;
1323      break;
1324    case kCondGE:
1325      true_high_cond = kCondGT;
1326      break;
1327    case kCondB:
1328      false_high_cond = kCondA;
1329      break;
1330    case kCondBE:
1331      true_high_cond = kCondB;
1332      break;
1333    case kCondA:
1334      false_high_cond = kCondB;
1335      break;
1336    case kCondAE:
1337      true_high_cond = kCondA;
1338      break;
1339  }
1340
1341  if (right.IsConstant()) {
1342    int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
1343    int32_t val_high = High32Bits(value);
1344    int32_t val_low = Low32Bits(value);
1345
1346    if (val_high == 0) {
1347      __ testl(left_high, left_high);
1348    } else {
1349      __ cmpl(left_high, Immediate(val_high));
1350    }
1351    if (if_cond == kCondNE) {
1352      __ j(X86Condition(true_high_cond), true_label);
1353    } else if (if_cond == kCondEQ) {
1354      __ j(X86Condition(false_high_cond), false_label);
1355    } else {
1356      __ j(X86Condition(true_high_cond), true_label);
1357      __ j(X86Condition(false_high_cond), false_label);
1358    }
1359    // Must be equal high, so compare the lows.
1360    if (val_low == 0) {
1361      __ testl(left_low, left_low);
1362    } else {
1363      __ cmpl(left_low, Immediate(val_low));
1364    }
1365  } else {
1366    Register right_high = right.AsRegisterPairHigh<Register>();
1367    Register right_low = right.AsRegisterPairLow<Register>();
1368
1369    __ cmpl(left_high, right_high);
1370    if (if_cond == kCondNE) {
1371      __ j(X86Condition(true_high_cond), true_label);
1372    } else if (if_cond == kCondEQ) {
1373      __ j(X86Condition(false_high_cond), false_label);
1374    } else {
1375      __ j(X86Condition(true_high_cond), true_label);
1376      __ j(X86Condition(false_high_cond), false_label);
1377    }
1378    // Must be equal high, so compare the lows.
1379    __ cmpl(left_low, right_low);
1380  }
1381  // The last comparison might be unsigned.
1382  __ j(final_condition, true_label);
1383}
1384
1385void InstructionCodeGeneratorX86::GenerateCompareTestAndBranch(HCondition* condition,
1386                                                               Label* true_target_in,
1387                                                               Label* false_target_in) {
1388  // Generated branching requires both targets to be explicit. If either of the
1389  // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead.
1390  Label fallthrough_target;
1391  Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in;
1392  Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in;
1393
1394  LocationSummary* locations = condition->GetLocations();
1395  Location left = locations->InAt(0);
1396  Location right = locations->InAt(1);
1397
1398  Primitive::Type type = condition->InputAt(0)->GetType();
1399  switch (type) {
1400    case Primitive::kPrimLong:
1401      GenerateLongComparesAndJumps(condition, true_target, false_target);
1402      break;
1403    case Primitive::kPrimFloat:
1404      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1405      GenerateFPJumps(condition, true_target, false_target);
1406      break;
1407    case Primitive::kPrimDouble:
1408      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
1409      GenerateFPJumps(condition, true_target, false_target);
1410      break;
1411    default:
1412      LOG(FATAL) << "Unexpected compare type " << type;
1413  }
1414
1415  if (false_target != &fallthrough_target) {
1416    __ jmp(false_target);
1417  }
1418
1419  if (fallthrough_target.IsLinked()) {
1420    __ Bind(&fallthrough_target);
1421  }
1422}
1423
1424static bool AreEflagsSetFrom(HInstruction* cond, HInstruction* branch) {
1425  // Moves may affect the eflags register (move zero uses xorl), so the EFLAGS
1426  // are set only strictly before `branch`. We can't use the eflags on long/FP
1427  // conditions if they are materialized due to the complex branching.
1428  return cond->IsCondition() &&
1429         cond->GetNext() == branch &&
1430         cond->InputAt(0)->GetType() != Primitive::kPrimLong &&
1431         !Primitive::IsFloatingPointType(cond->InputAt(0)->GetType());
1432}
1433
1434void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instruction,
1435                                                        size_t condition_input_index,
1436                                                        Label* true_target,
1437                                                        Label* false_target) {
1438  HInstruction* cond = instruction->InputAt(condition_input_index);
1439
1440  if (true_target == nullptr && false_target == nullptr) {
1441    // Nothing to do. The code always falls through.
1442    return;
1443  } else if (cond->IsIntConstant()) {
1444    // Constant condition, statically compared against 1.
1445    if (cond->AsIntConstant()->IsOne()) {
1446      if (true_target != nullptr) {
1447        __ jmp(true_target);
1448      }
1449    } else {
1450      DCHECK(cond->AsIntConstant()->IsZero());
1451      if (false_target != nullptr) {
1452        __ jmp(false_target);
1453      }
1454    }
1455    return;
1456  }
1457
1458  // The following code generates these patterns:
1459  //  (1) true_target == nullptr && false_target != nullptr
1460  //        - opposite condition true => branch to false_target
1461  //  (2) true_target != nullptr && false_target == nullptr
1462  //        - condition true => branch to true_target
1463  //  (3) true_target != nullptr && false_target != nullptr
1464  //        - condition true => branch to true_target
1465  //        - branch to false_target
1466  if (IsBooleanValueOrMaterializedCondition(cond)) {
1467    if (AreEflagsSetFrom(cond, instruction)) {
1468      if (true_target == nullptr) {
1469        __ j(X86Condition(cond->AsCondition()->GetOppositeCondition()), false_target);
1470      } else {
1471        __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
1472      }
1473    } else {
1474      // Materialized condition, compare against 0.
1475      Location lhs = instruction->GetLocations()->InAt(condition_input_index);
1476      if (lhs.IsRegister()) {
1477        __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1478      } else {
1479        __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
1480      }
1481      if (true_target == nullptr) {
1482        __ j(kEqual, false_target);
1483      } else {
1484        __ j(kNotEqual, true_target);
1485      }
1486    }
1487  } else {
1488    // Condition has not been materialized, use its inputs as the comparison and
1489    // its condition as the branch condition.
1490    HCondition* condition = cond->AsCondition();
1491
1492    // If this is a long or FP comparison that has been folded into
1493    // the HCondition, generate the comparison directly.
1494    Primitive::Type type = condition->InputAt(0)->GetType();
1495    if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
1496      GenerateCompareTestAndBranch(condition, true_target, false_target);
1497      return;
1498    }
1499
1500    Location lhs = condition->GetLocations()->InAt(0);
1501    Location rhs = condition->GetLocations()->InAt(1);
1502    // LHS is guaranteed to be in a register (see LocationsBuilderX86::VisitCondition).
1503    if (rhs.IsRegister()) {
1504      __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1505    } else if (rhs.IsConstant()) {
1506      int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1507      if (constant == 0) {
1508        __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1509      } else {
1510        __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1511      }
1512    } else {
1513      __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1514    }
1515    if (true_target == nullptr) {
1516      __ j(X86Condition(condition->GetOppositeCondition()), false_target);
1517    } else {
1518      __ j(X86Condition(condition->GetCondition()), true_target);
1519    }
1520  }
1521
1522  // If neither branch falls through (case 3), the conditional branch to `true_target`
1523  // was already emitted (case 2) and we need to emit a jump to `false_target`.
1524  if (true_target != nullptr && false_target != nullptr) {
1525    __ jmp(false_target);
1526  }
1527}
1528
1529void LocationsBuilderX86::VisitIf(HIf* if_instr) {
1530  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
1531  if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
1532    locations->SetInAt(0, Location::Any());
1533  }
1534}
1535
1536void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
1537  HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
1538  HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
1539  Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
1540      nullptr : codegen_->GetLabelOf(true_successor);
1541  Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
1542      nullptr : codegen_->GetLabelOf(false_successor);
1543  GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
1544}
1545
1546void LocationsBuilderX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1547  LocationSummary* locations = new (GetGraph()->GetArena())
1548      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
1549  if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
1550    locations->SetInAt(0, Location::Any());
1551  }
1552}
1553
1554void InstructionCodeGeneratorX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1555  SlowPathCode* slow_path = new (GetGraph()->GetArena())
1556      DeoptimizationSlowPathX86(deoptimize);
1557  codegen_->AddSlowPath(slow_path);
1558  GenerateTestAndBranch(deoptimize,
1559                        /* condition_input_index */ 0,
1560                        slow_path->GetEntryLabel(),
1561                        /* false_target */ nullptr);
1562}
1563
1564void LocationsBuilderX86::VisitLocal(HLocal* local) {
1565  local->SetLocations(nullptr);
1566}
1567
1568void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
1569  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
1570}
1571
1572void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
1573  local->SetLocations(nullptr);
1574}
1575
1576void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_UNUSED) {
1577  // Nothing to do, this is driven by the code generator.
1578}
1579
1580void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
1581  LocationSummary* locations =
1582      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
1583  switch (store->InputAt(1)->GetType()) {
1584    case Primitive::kPrimBoolean:
1585    case Primitive::kPrimByte:
1586    case Primitive::kPrimChar:
1587    case Primitive::kPrimShort:
1588    case Primitive::kPrimInt:
1589    case Primitive::kPrimNot:
1590    case Primitive::kPrimFloat:
1591      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
1592      break;
1593
1594    case Primitive::kPrimLong:
1595    case Primitive::kPrimDouble:
1596      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
1597      break;
1598
1599    default:
1600      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
1601  }
1602}
1603
1604void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store ATTRIBUTE_UNUSED) {
1605}
1606
1607void LocationsBuilderX86::VisitCondition(HCondition* cond) {
1608  LocationSummary* locations =
1609      new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
1610  // Handle the long/FP comparisons made in instruction simplification.
1611  switch (cond->InputAt(0)->GetType()) {
1612    case Primitive::kPrimLong: {
1613      locations->SetInAt(0, Location::RequiresRegister());
1614      locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1)));
1615      if (cond->NeedsMaterialization()) {
1616        locations->SetOut(Location::RequiresRegister());
1617      }
1618      break;
1619    }
1620    case Primitive::kPrimFloat:
1621    case Primitive::kPrimDouble: {
1622      locations->SetInAt(0, Location::RequiresFpuRegister());
1623      locations->SetInAt(1, Location::RequiresFpuRegister());
1624      if (cond->NeedsMaterialization()) {
1625        locations->SetOut(Location::RequiresRegister());
1626      }
1627      break;
1628    }
1629    default:
1630      locations->SetInAt(0, Location::RequiresRegister());
1631      locations->SetInAt(1, Location::Any());
1632      if (cond->NeedsMaterialization()) {
1633        // We need a byte register.
1634        locations->SetOut(Location::RegisterLocation(ECX));
1635      }
1636      break;
1637  }
1638}
1639
1640void InstructionCodeGeneratorX86::VisitCondition(HCondition* cond) {
1641  if (!cond->NeedsMaterialization()) {
1642    return;
1643  }
1644
1645  LocationSummary* locations = cond->GetLocations();
1646  Location lhs = locations->InAt(0);
1647  Location rhs = locations->InAt(1);
1648  Register reg = locations->Out().AsRegister<Register>();
1649  Label true_label, false_label;
1650
1651  switch (cond->InputAt(0)->GetType()) {
1652    default: {
1653      // Integer case.
1654
1655      // Clear output register: setb only sets the low byte.
1656      __ xorl(reg, reg);
1657
1658      if (rhs.IsRegister()) {
1659        __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1660      } else if (rhs.IsConstant()) {
1661        int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1662        if (constant == 0) {
1663          __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1664        } else {
1665          __ cmpl(lhs.AsRegister<Register>(), Immediate(constant));
1666        }
1667      } else {
1668        __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1669      }
1670      __ setb(X86Condition(cond->GetCondition()), reg);
1671      return;
1672    }
1673    case Primitive::kPrimLong:
1674      GenerateLongComparesAndJumps(cond, &true_label, &false_label);
1675      break;
1676    case Primitive::kPrimFloat:
1677      __ ucomiss(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1678      GenerateFPJumps(cond, &true_label, &false_label);
1679      break;
1680    case Primitive::kPrimDouble:
1681      __ ucomisd(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1682      GenerateFPJumps(cond, &true_label, &false_label);
1683      break;
1684  }
1685
1686  // Convert the jumps into the result.
1687  NearLabel done_label;
1688
1689  // False case: result = 0.
1690  __ Bind(&false_label);
1691  __ xorl(reg, reg);
1692  __ jmp(&done_label);
1693
1694  // True case: result = 1.
1695  __ Bind(&true_label);
1696  __ movl(reg, Immediate(1));
1697  __ Bind(&done_label);
1698}
1699
1700void LocationsBuilderX86::VisitEqual(HEqual* comp) {
1701  VisitCondition(comp);
1702}
1703
1704void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
1705  VisitCondition(comp);
1706}
1707
1708void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
1709  VisitCondition(comp);
1710}
1711
1712void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
1713  VisitCondition(comp);
1714}
1715
1716void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
1717  VisitCondition(comp);
1718}
1719
1720void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
1721  VisitCondition(comp);
1722}
1723
1724void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1725  VisitCondition(comp);
1726}
1727
1728void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1729  VisitCondition(comp);
1730}
1731
1732void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
1733  VisitCondition(comp);
1734}
1735
1736void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
1737  VisitCondition(comp);
1738}
1739
1740void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1741  VisitCondition(comp);
1742}
1743
1744void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1745  VisitCondition(comp);
1746}
1747
1748void LocationsBuilderX86::VisitBelow(HBelow* comp) {
1749  VisitCondition(comp);
1750}
1751
1752void InstructionCodeGeneratorX86::VisitBelow(HBelow* comp) {
1753  VisitCondition(comp);
1754}
1755
1756void LocationsBuilderX86::VisitBelowOrEqual(HBelowOrEqual* comp) {
1757  VisitCondition(comp);
1758}
1759
1760void InstructionCodeGeneratorX86::VisitBelowOrEqual(HBelowOrEqual* comp) {
1761  VisitCondition(comp);
1762}
1763
1764void LocationsBuilderX86::VisitAbove(HAbove* comp) {
1765  VisitCondition(comp);
1766}
1767
1768void InstructionCodeGeneratorX86::VisitAbove(HAbove* comp) {
1769  VisitCondition(comp);
1770}
1771
1772void LocationsBuilderX86::VisitAboveOrEqual(HAboveOrEqual* comp) {
1773  VisitCondition(comp);
1774}
1775
1776void InstructionCodeGeneratorX86::VisitAboveOrEqual(HAboveOrEqual* comp) {
1777  VisitCondition(comp);
1778}
1779
1780void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1781  LocationSummary* locations =
1782      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1783  locations->SetOut(Location::ConstantLocation(constant));
1784}
1785
1786void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) {
1787  // Will be generated at use site.
1788}
1789
1790void LocationsBuilderX86::VisitNullConstant(HNullConstant* constant) {
1791  LocationSummary* locations =
1792      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1793  locations->SetOut(Location::ConstantLocation(constant));
1794}
1795
1796void InstructionCodeGeneratorX86::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) {
1797  // Will be generated at use site.
1798}
1799
1800void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1801  LocationSummary* locations =
1802      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1803  locations->SetOut(Location::ConstantLocation(constant));
1804}
1805
1806void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) {
1807  // Will be generated at use site.
1808}
1809
1810void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1811  LocationSummary* locations =
1812      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1813  locations->SetOut(Location::ConstantLocation(constant));
1814}
1815
1816void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) {
1817  // Will be generated at use site.
1818}
1819
1820void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1821  LocationSummary* locations =
1822      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1823  locations->SetOut(Location::ConstantLocation(constant));
1824}
1825
1826void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) {
1827  // Will be generated at use site.
1828}
1829
1830void LocationsBuilderX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1831  memory_barrier->SetLocations(nullptr);
1832}
1833
1834void InstructionCodeGeneratorX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1835  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1836}
1837
1838void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1839  ret->SetLocations(nullptr);
1840}
1841
1842void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) {
1843  codegen_->GenerateFrameExit();
1844}
1845
1846void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1847  LocationSummary* locations =
1848      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1849  switch (ret->InputAt(0)->GetType()) {
1850    case Primitive::kPrimBoolean:
1851    case Primitive::kPrimByte:
1852    case Primitive::kPrimChar:
1853    case Primitive::kPrimShort:
1854    case Primitive::kPrimInt:
1855    case Primitive::kPrimNot:
1856      locations->SetInAt(0, Location::RegisterLocation(EAX));
1857      break;
1858
1859    case Primitive::kPrimLong:
1860      locations->SetInAt(
1861          0, Location::RegisterPairLocation(EAX, EDX));
1862      break;
1863
1864    case Primitive::kPrimFloat:
1865    case Primitive::kPrimDouble:
1866      locations->SetInAt(
1867          0, Location::FpuRegisterLocation(XMM0));
1868      break;
1869
1870    default:
1871      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1872  }
1873}
1874
1875void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1876  if (kIsDebugBuild) {
1877    switch (ret->InputAt(0)->GetType()) {
1878      case Primitive::kPrimBoolean:
1879      case Primitive::kPrimByte:
1880      case Primitive::kPrimChar:
1881      case Primitive::kPrimShort:
1882      case Primitive::kPrimInt:
1883      case Primitive::kPrimNot:
1884        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1885        break;
1886
1887      case Primitive::kPrimLong:
1888        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1889        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1890        break;
1891
1892      case Primitive::kPrimFloat:
1893      case Primitive::kPrimDouble:
1894        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1895        break;
1896
1897      default:
1898        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1899    }
1900  }
1901  codegen_->GenerateFrameExit();
1902}
1903
1904void LocationsBuilderX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1905  // The trampoline uses the same calling convention as dex calling conventions,
1906  // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain
1907  // the method_idx.
1908  HandleInvoke(invoke);
1909}
1910
1911void InstructionCodeGeneratorX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1912  codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
1913}
1914
1915void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1916  // When we do not run baseline, explicit clinit checks triggered by static
1917  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1918  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1919
1920  IntrinsicLocationsBuilderX86 intrinsic(codegen_);
1921  if (intrinsic.TryDispatch(invoke)) {
1922    if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) {
1923      invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any());
1924    }
1925    return;
1926  }
1927
1928  HandleInvoke(invoke);
1929
1930  // For PC-relative dex cache the invoke has an extra input, the PC-relative address base.
1931  if (invoke->HasPcRelativeDexCache()) {
1932    invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister());
1933  }
1934
1935  if (codegen_->IsBaseline()) {
1936    // Baseline does not have enough registers if the current method also
1937    // needs a register. We therefore do not require a register for it, and let
1938    // the code generation of the invoke handle it.
1939    LocationSummary* locations = invoke->GetLocations();
1940    Location location = locations->InAt(invoke->GetSpecialInputIndex());
1941    if (location.IsUnallocated() && location.GetPolicy() == Location::kRequiresRegister) {
1942      locations->SetInAt(invoke->GetSpecialInputIndex(), Location::NoLocation());
1943    }
1944  }
1945}
1946
1947static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) {
1948  if (invoke->GetLocations()->Intrinsified()) {
1949    IntrinsicCodeGeneratorX86 intrinsic(codegen);
1950    intrinsic.Dispatch(invoke);
1951    return true;
1952  }
1953  return false;
1954}
1955
1956void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1957  // When we do not run baseline, explicit clinit checks triggered by static
1958  // invokes must have been pruned by art::PrepareForRegisterAllocation.
1959  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1960
1961  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1962    return;
1963  }
1964
1965  LocationSummary* locations = invoke->GetLocations();
1966  codegen_->GenerateStaticOrDirectCall(
1967      invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
1968  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1969}
1970
1971void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1972  HandleInvoke(invoke);
1973}
1974
1975void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
1976  InvokeDexCallingConventionVisitorX86 calling_convention_visitor;
1977  CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
1978}
1979
1980void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1981  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1982    return;
1983  }
1984
1985  codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
1986  DCHECK(!codegen_->IsLeafMethod());
1987  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1988}
1989
1990void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
1991  // This call to HandleInvoke allocates a temporary (core) register
1992  // which is also used to transfer the hidden argument from FP to
1993  // core register.
1994  HandleInvoke(invoke);
1995  // Add the hidden argument.
1996  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
1997}
1998
1999void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
2000  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
2001  LocationSummary* locations = invoke->GetLocations();
2002  Register temp = locations->GetTemp(0).AsRegister<Register>();
2003  XmmRegister hidden_reg = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
2004  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
2005      invoke->GetImtIndex() % mirror::Class::kImtSize, kX86PointerSize).Uint32Value();
2006  Location receiver = locations->InAt(0);
2007  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
2008
2009  // Set the hidden argument. This is safe to do this here, as XMM7
2010  // won't be modified thereafter, before the `call` instruction.
2011  DCHECK_EQ(XMM7, hidden_reg);
2012  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
2013  __ movd(hidden_reg, temp);
2014
2015  if (receiver.IsStackSlot()) {
2016    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
2017    // /* HeapReference<Class> */ temp = temp->klass_
2018    __ movl(temp, Address(temp, class_offset));
2019  } else {
2020    // /* HeapReference<Class> */ temp = receiver->klass_
2021    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
2022  }
2023  codegen_->MaybeRecordImplicitNullCheck(invoke);
2024  // Instead of simply (possibly) unpoisoning `temp` here, we should
2025  // emit a read barrier for the previous class reference load.
2026  // However this is not required in practice, as this is an
2027  // intermediate/temporary reference and because the current
2028  // concurrent copying collector keeps the from-space memory
2029  // intact/accessible until the end of the marking phase (the
2030  // concurrent copying collector may not in the future).
2031  __ MaybeUnpoisonHeapReference(temp);
2032  // temp = temp->GetImtEntryAt(method_offset);
2033  __ movl(temp, Address(temp, method_offset));
2034  // call temp->GetEntryPoint();
2035  __ call(Address(temp,
2036                  ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
2037
2038  DCHECK(!codegen_->IsLeafMethod());
2039  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2040}
2041
2042void LocationsBuilderX86::VisitNeg(HNeg* neg) {
2043  LocationSummary* locations =
2044      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
2045  switch (neg->GetResultType()) {
2046    case Primitive::kPrimInt:
2047    case Primitive::kPrimLong:
2048      locations->SetInAt(0, Location::RequiresRegister());
2049      locations->SetOut(Location::SameAsFirstInput());
2050      break;
2051
2052    case Primitive::kPrimFloat:
2053      locations->SetInAt(0, Location::RequiresFpuRegister());
2054      locations->SetOut(Location::SameAsFirstInput());
2055      locations->AddTemp(Location::RequiresRegister());
2056      locations->AddTemp(Location::RequiresFpuRegister());
2057      break;
2058
2059    case Primitive::kPrimDouble:
2060      locations->SetInAt(0, Location::RequiresFpuRegister());
2061      locations->SetOut(Location::SameAsFirstInput());
2062      locations->AddTemp(Location::RequiresFpuRegister());
2063      break;
2064
2065    default:
2066      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
2067  }
2068}
2069
2070void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
2071  LocationSummary* locations = neg->GetLocations();
2072  Location out = locations->Out();
2073  Location in = locations->InAt(0);
2074  switch (neg->GetResultType()) {
2075    case Primitive::kPrimInt:
2076      DCHECK(in.IsRegister());
2077      DCHECK(in.Equals(out));
2078      __ negl(out.AsRegister<Register>());
2079      break;
2080
2081    case Primitive::kPrimLong:
2082      DCHECK(in.IsRegisterPair());
2083      DCHECK(in.Equals(out));
2084      __ negl(out.AsRegisterPairLow<Register>());
2085      // Negation is similar to subtraction from zero.  The least
2086      // significant byte triggers a borrow when it is different from
2087      // zero; to take it into account, add 1 to the most significant
2088      // byte if the carry flag (CF) is set to 1 after the first NEGL
2089      // operation.
2090      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
2091      __ negl(out.AsRegisterPairHigh<Register>());
2092      break;
2093
2094    case Primitive::kPrimFloat: {
2095      DCHECK(in.Equals(out));
2096      Register constant = locations->GetTemp(0).AsRegister<Register>();
2097      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
2098      // Implement float negation with an exclusive or with value
2099      // 0x80000000 (mask for bit 31, representing the sign of a
2100      // single-precision floating-point number).
2101      __ movl(constant, Immediate(INT32_C(0x80000000)));
2102      __ movd(mask, constant);
2103      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
2104      break;
2105    }
2106
2107    case Primitive::kPrimDouble: {
2108      DCHECK(in.Equals(out));
2109      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2110      // Implement double negation with an exclusive or with value
2111      // 0x8000000000000000 (mask for bit 63, representing the sign of
2112      // a double-precision floating-point number).
2113      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
2114      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
2115      break;
2116    }
2117
2118    default:
2119      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
2120  }
2121}
2122
2123void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
2124  Primitive::Type result_type = conversion->GetResultType();
2125  Primitive::Type input_type = conversion->GetInputType();
2126  DCHECK_NE(result_type, input_type);
2127
2128  // The float-to-long and double-to-long type conversions rely on a
2129  // call to the runtime.
2130  LocationSummary::CallKind call_kind =
2131      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
2132       && result_type == Primitive::kPrimLong)
2133      ? LocationSummary::kCall
2134      : LocationSummary::kNoCall;
2135  LocationSummary* locations =
2136      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
2137
2138  // The Java language does not allow treating boolean as an integral type but
2139  // our bit representation makes it safe.
2140
2141  switch (result_type) {
2142    case Primitive::kPrimByte:
2143      switch (input_type) {
2144        case Primitive::kPrimBoolean:
2145          // Boolean input is a result of code transformations.
2146        case Primitive::kPrimShort:
2147        case Primitive::kPrimInt:
2148        case Primitive::kPrimChar:
2149          // Processing a Dex `int-to-byte' instruction.
2150          locations->SetInAt(0, Location::ByteRegisterOrConstant(ECX, conversion->InputAt(0)));
2151          // Make the output overlap to please the register allocator. This greatly simplifies
2152          // the validation of the linear scan implementation
2153          locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
2154          break;
2155
2156        default:
2157          LOG(FATAL) << "Unexpected type conversion from " << input_type
2158                     << " to " << result_type;
2159      }
2160      break;
2161
2162    case Primitive::kPrimShort:
2163      switch (input_type) {
2164        case Primitive::kPrimBoolean:
2165          // Boolean input is a result of code transformations.
2166        case Primitive::kPrimByte:
2167        case Primitive::kPrimInt:
2168        case Primitive::kPrimChar:
2169          // Processing a Dex `int-to-short' instruction.
2170          locations->SetInAt(0, Location::Any());
2171          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2172          break;
2173
2174        default:
2175          LOG(FATAL) << "Unexpected type conversion from " << input_type
2176                     << " to " << result_type;
2177      }
2178      break;
2179
2180    case Primitive::kPrimInt:
2181      switch (input_type) {
2182        case Primitive::kPrimLong:
2183          // Processing a Dex `long-to-int' instruction.
2184          locations->SetInAt(0, Location::Any());
2185          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2186          break;
2187
2188        case Primitive::kPrimFloat:
2189          // Processing a Dex `float-to-int' instruction.
2190          locations->SetInAt(0, Location::RequiresFpuRegister());
2191          locations->SetOut(Location::RequiresRegister());
2192          locations->AddTemp(Location::RequiresFpuRegister());
2193          break;
2194
2195        case Primitive::kPrimDouble:
2196          // Processing a Dex `double-to-int' instruction.
2197          locations->SetInAt(0, Location::RequiresFpuRegister());
2198          locations->SetOut(Location::RequiresRegister());
2199          locations->AddTemp(Location::RequiresFpuRegister());
2200          break;
2201
2202        default:
2203          LOG(FATAL) << "Unexpected type conversion from " << input_type
2204                     << " to " << result_type;
2205      }
2206      break;
2207
2208    case Primitive::kPrimLong:
2209      switch (input_type) {
2210        case Primitive::kPrimBoolean:
2211          // Boolean input is a result of code transformations.
2212        case Primitive::kPrimByte:
2213        case Primitive::kPrimShort:
2214        case Primitive::kPrimInt:
2215        case Primitive::kPrimChar:
2216          // Processing a Dex `int-to-long' instruction.
2217          locations->SetInAt(0, Location::RegisterLocation(EAX));
2218          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2219          break;
2220
2221        case Primitive::kPrimFloat:
2222        case Primitive::kPrimDouble: {
2223          // Processing a Dex `float-to-long' or 'double-to-long' instruction.
2224          InvokeRuntimeCallingConvention calling_convention;
2225          XmmRegister parameter = calling_convention.GetFpuRegisterAt(0);
2226          locations->SetInAt(0, Location::FpuRegisterLocation(parameter));
2227
2228          // The runtime helper puts the result in EAX, EDX.
2229          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2230        }
2231        break;
2232
2233        default:
2234          LOG(FATAL) << "Unexpected type conversion from " << input_type
2235                     << " to " << result_type;
2236      }
2237      break;
2238
2239    case Primitive::kPrimChar:
2240      switch (input_type) {
2241        case Primitive::kPrimBoolean:
2242          // Boolean input is a result of code transformations.
2243        case Primitive::kPrimByte:
2244        case Primitive::kPrimShort:
2245        case Primitive::kPrimInt:
2246          // Processing a Dex `int-to-char' instruction.
2247          locations->SetInAt(0, Location::Any());
2248          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2249          break;
2250
2251        default:
2252          LOG(FATAL) << "Unexpected type conversion from " << input_type
2253                     << " to " << result_type;
2254      }
2255      break;
2256
2257    case Primitive::kPrimFloat:
2258      switch (input_type) {
2259        case Primitive::kPrimBoolean:
2260          // Boolean input is a result of code transformations.
2261        case Primitive::kPrimByte:
2262        case Primitive::kPrimShort:
2263        case Primitive::kPrimInt:
2264        case Primitive::kPrimChar:
2265          // Processing a Dex `int-to-float' instruction.
2266          locations->SetInAt(0, Location::RequiresRegister());
2267          locations->SetOut(Location::RequiresFpuRegister());
2268          break;
2269
2270        case Primitive::kPrimLong:
2271          // Processing a Dex `long-to-float' instruction.
2272          locations->SetInAt(0, Location::Any());
2273          locations->SetOut(Location::Any());
2274          break;
2275
2276        case Primitive::kPrimDouble:
2277          // Processing a Dex `double-to-float' instruction.
2278          locations->SetInAt(0, Location::RequiresFpuRegister());
2279          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2280          break;
2281
2282        default:
2283          LOG(FATAL) << "Unexpected type conversion from " << input_type
2284                     << " to " << result_type;
2285      };
2286      break;
2287
2288    case Primitive::kPrimDouble:
2289      switch (input_type) {
2290        case Primitive::kPrimBoolean:
2291          // Boolean input is a result of code transformations.
2292        case Primitive::kPrimByte:
2293        case Primitive::kPrimShort:
2294        case Primitive::kPrimInt:
2295        case Primitive::kPrimChar:
2296          // Processing a Dex `int-to-double' instruction.
2297          locations->SetInAt(0, Location::RequiresRegister());
2298          locations->SetOut(Location::RequiresFpuRegister());
2299          break;
2300
2301        case Primitive::kPrimLong:
2302          // Processing a Dex `long-to-double' instruction.
2303          locations->SetInAt(0, Location::Any());
2304          locations->SetOut(Location::Any());
2305          break;
2306
2307        case Primitive::kPrimFloat:
2308          // Processing a Dex `float-to-double' instruction.
2309          locations->SetInAt(0, Location::RequiresFpuRegister());
2310          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2311          break;
2312
2313        default:
2314          LOG(FATAL) << "Unexpected type conversion from " << input_type
2315                     << " to " << result_type;
2316      }
2317      break;
2318
2319    default:
2320      LOG(FATAL) << "Unexpected type conversion from " << input_type
2321                 << " to " << result_type;
2322  }
2323}
2324
2325void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
2326  LocationSummary* locations = conversion->GetLocations();
2327  Location out = locations->Out();
2328  Location in = locations->InAt(0);
2329  Primitive::Type result_type = conversion->GetResultType();
2330  Primitive::Type input_type = conversion->GetInputType();
2331  DCHECK_NE(result_type, input_type);
2332  switch (result_type) {
2333    case Primitive::kPrimByte:
2334      switch (input_type) {
2335        case Primitive::kPrimBoolean:
2336          // Boolean input is a result of code transformations.
2337        case Primitive::kPrimShort:
2338        case Primitive::kPrimInt:
2339        case Primitive::kPrimChar:
2340          // Processing a Dex `int-to-byte' instruction.
2341          if (in.IsRegister()) {
2342            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
2343          } else {
2344            DCHECK(in.GetConstant()->IsIntConstant());
2345            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2346            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
2347          }
2348          break;
2349
2350        default:
2351          LOG(FATAL) << "Unexpected type conversion from " << input_type
2352                     << " to " << result_type;
2353      }
2354      break;
2355
2356    case Primitive::kPrimShort:
2357      switch (input_type) {
2358        case Primitive::kPrimBoolean:
2359          // Boolean input is a result of code transformations.
2360        case Primitive::kPrimByte:
2361        case Primitive::kPrimInt:
2362        case Primitive::kPrimChar:
2363          // Processing a Dex `int-to-short' instruction.
2364          if (in.IsRegister()) {
2365            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2366          } else if (in.IsStackSlot()) {
2367            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2368          } else {
2369            DCHECK(in.GetConstant()->IsIntConstant());
2370            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2371            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
2372          }
2373          break;
2374
2375        default:
2376          LOG(FATAL) << "Unexpected type conversion from " << input_type
2377                     << " to " << result_type;
2378      }
2379      break;
2380
2381    case Primitive::kPrimInt:
2382      switch (input_type) {
2383        case Primitive::kPrimLong:
2384          // Processing a Dex `long-to-int' instruction.
2385          if (in.IsRegisterPair()) {
2386            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
2387          } else if (in.IsDoubleStackSlot()) {
2388            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2389          } else {
2390            DCHECK(in.IsConstant());
2391            DCHECK(in.GetConstant()->IsLongConstant());
2392            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
2393            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
2394          }
2395          break;
2396
2397        case Primitive::kPrimFloat: {
2398          // Processing a Dex `float-to-int' instruction.
2399          XmmRegister input = in.AsFpuRegister<XmmRegister>();
2400          Register output = out.AsRegister<Register>();
2401          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2402          NearLabel done, nan;
2403
2404          __ movl(output, Immediate(kPrimIntMax));
2405          // temp = int-to-float(output)
2406          __ cvtsi2ss(temp, output);
2407          // if input >= temp goto done
2408          __ comiss(input, temp);
2409          __ j(kAboveEqual, &done);
2410          // if input == NaN goto nan
2411          __ j(kUnordered, &nan);
2412          // output = float-to-int-truncate(input)
2413          __ cvttss2si(output, input);
2414          __ jmp(&done);
2415          __ Bind(&nan);
2416          //  output = 0
2417          __ xorl(output, output);
2418          __ Bind(&done);
2419          break;
2420        }
2421
2422        case Primitive::kPrimDouble: {
2423          // Processing a Dex `double-to-int' instruction.
2424          XmmRegister input = in.AsFpuRegister<XmmRegister>();
2425          Register output = out.AsRegister<Register>();
2426          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2427          NearLabel done, nan;
2428
2429          __ movl(output, Immediate(kPrimIntMax));
2430          // temp = int-to-double(output)
2431          __ cvtsi2sd(temp, output);
2432          // if input >= temp goto done
2433          __ comisd(input, temp);
2434          __ j(kAboveEqual, &done);
2435          // if input == NaN goto nan
2436          __ j(kUnordered, &nan);
2437          // output = double-to-int-truncate(input)
2438          __ cvttsd2si(output, input);
2439          __ jmp(&done);
2440          __ Bind(&nan);
2441          //  output = 0
2442          __ xorl(output, output);
2443          __ Bind(&done);
2444          break;
2445        }
2446
2447        default:
2448          LOG(FATAL) << "Unexpected type conversion from " << input_type
2449                     << " to " << result_type;
2450      }
2451      break;
2452
2453    case Primitive::kPrimLong:
2454      switch (input_type) {
2455        case Primitive::kPrimBoolean:
2456          // Boolean input is a result of code transformations.
2457        case Primitive::kPrimByte:
2458        case Primitive::kPrimShort:
2459        case Primitive::kPrimInt:
2460        case Primitive::kPrimChar:
2461          // Processing a Dex `int-to-long' instruction.
2462          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
2463          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
2464          DCHECK_EQ(in.AsRegister<Register>(), EAX);
2465          __ cdq();
2466          break;
2467
2468        case Primitive::kPrimFloat:
2469          // Processing a Dex `float-to-long' instruction.
2470          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pF2l),
2471                                  conversion,
2472                                  conversion->GetDexPc(),
2473                                  nullptr);
2474          CheckEntrypointTypes<kQuickF2l, int64_t, float>();
2475          break;
2476
2477        case Primitive::kPrimDouble:
2478          // Processing a Dex `double-to-long' instruction.
2479          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pD2l),
2480                                  conversion,
2481                                  conversion->GetDexPc(),
2482                                  nullptr);
2483          CheckEntrypointTypes<kQuickD2l, int64_t, double>();
2484          break;
2485
2486        default:
2487          LOG(FATAL) << "Unexpected type conversion from " << input_type
2488                     << " to " << result_type;
2489      }
2490      break;
2491
2492    case Primitive::kPrimChar:
2493      switch (input_type) {
2494        case Primitive::kPrimBoolean:
2495          // Boolean input is a result of code transformations.
2496        case Primitive::kPrimByte:
2497        case Primitive::kPrimShort:
2498        case Primitive::kPrimInt:
2499          // Processing a Dex `Process a Dex `int-to-char'' instruction.
2500          if (in.IsRegister()) {
2501            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2502          } else if (in.IsStackSlot()) {
2503            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2504          } else {
2505            DCHECK(in.GetConstant()->IsIntConstant());
2506            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2507            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
2508          }
2509          break;
2510
2511        default:
2512          LOG(FATAL) << "Unexpected type conversion from " << input_type
2513                     << " to " << result_type;
2514      }
2515      break;
2516
2517    case Primitive::kPrimFloat:
2518      switch (input_type) {
2519        case Primitive::kPrimBoolean:
2520          // Boolean input is a result of code transformations.
2521        case Primitive::kPrimByte:
2522        case Primitive::kPrimShort:
2523        case Primitive::kPrimInt:
2524        case Primitive::kPrimChar:
2525          // Processing a Dex `int-to-float' instruction.
2526          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2527          break;
2528
2529        case Primitive::kPrimLong: {
2530          // Processing a Dex `long-to-float' instruction.
2531          size_t adjustment = 0;
2532
2533          // Create stack space for the call to
2534          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstps below.
2535          // TODO: enhance register allocator to ask for stack temporaries.
2536          if (!in.IsDoubleStackSlot() || !out.IsStackSlot()) {
2537            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2538            __ subl(ESP, Immediate(adjustment));
2539          }
2540
2541          // Load the value to the FP stack, using temporaries if needed.
2542          PushOntoFPStack(in, 0, adjustment, false, true);
2543
2544          if (out.IsStackSlot()) {
2545            __ fstps(Address(ESP, out.GetStackIndex() + adjustment));
2546          } else {
2547            __ fstps(Address(ESP, 0));
2548            Location stack_temp = Location::StackSlot(0);
2549            codegen_->Move32(out, stack_temp);
2550          }
2551
2552          // Remove the temporary stack space we allocated.
2553          if (adjustment != 0) {
2554            __ addl(ESP, Immediate(adjustment));
2555          }
2556          break;
2557        }
2558
2559        case Primitive::kPrimDouble:
2560          // Processing a Dex `double-to-float' instruction.
2561          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2562          break;
2563
2564        default:
2565          LOG(FATAL) << "Unexpected type conversion from " << input_type
2566                     << " to " << result_type;
2567      };
2568      break;
2569
2570    case Primitive::kPrimDouble:
2571      switch (input_type) {
2572        case Primitive::kPrimBoolean:
2573          // Boolean input is a result of code transformations.
2574        case Primitive::kPrimByte:
2575        case Primitive::kPrimShort:
2576        case Primitive::kPrimInt:
2577        case Primitive::kPrimChar:
2578          // Processing a Dex `int-to-double' instruction.
2579          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2580          break;
2581
2582        case Primitive::kPrimLong: {
2583          // Processing a Dex `long-to-double' instruction.
2584          size_t adjustment = 0;
2585
2586          // Create stack space for the call to
2587          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstpl below.
2588          // TODO: enhance register allocator to ask for stack temporaries.
2589          if (!in.IsDoubleStackSlot() || !out.IsDoubleStackSlot()) {
2590            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2591            __ subl(ESP, Immediate(adjustment));
2592          }
2593
2594          // Load the value to the FP stack, using temporaries if needed.
2595          PushOntoFPStack(in, 0, adjustment, false, true);
2596
2597          if (out.IsDoubleStackSlot()) {
2598            __ fstpl(Address(ESP, out.GetStackIndex() + adjustment));
2599          } else {
2600            __ fstpl(Address(ESP, 0));
2601            Location stack_temp = Location::DoubleStackSlot(0);
2602            codegen_->Move64(out, stack_temp);
2603          }
2604
2605          // Remove the temporary stack space we allocated.
2606          if (adjustment != 0) {
2607            __ addl(ESP, Immediate(adjustment));
2608          }
2609          break;
2610        }
2611
2612        case Primitive::kPrimFloat:
2613          // Processing a Dex `float-to-double' instruction.
2614          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2615          break;
2616
2617        default:
2618          LOG(FATAL) << "Unexpected type conversion from " << input_type
2619                     << " to " << result_type;
2620      };
2621      break;
2622
2623    default:
2624      LOG(FATAL) << "Unexpected type conversion from " << input_type
2625                 << " to " << result_type;
2626  }
2627}
2628
2629void LocationsBuilderX86::VisitAdd(HAdd* add) {
2630  LocationSummary* locations =
2631      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
2632  switch (add->GetResultType()) {
2633    case Primitive::kPrimInt: {
2634      locations->SetInAt(0, Location::RequiresRegister());
2635      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
2636      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2637      break;
2638    }
2639
2640    case Primitive::kPrimLong: {
2641      locations->SetInAt(0, Location::RequiresRegister());
2642      locations->SetInAt(1, Location::Any());
2643      locations->SetOut(Location::SameAsFirstInput());
2644      break;
2645    }
2646
2647    case Primitive::kPrimFloat:
2648    case Primitive::kPrimDouble: {
2649      locations->SetInAt(0, Location::RequiresFpuRegister());
2650      locations->SetInAt(1, Location::Any());
2651      locations->SetOut(Location::SameAsFirstInput());
2652      break;
2653    }
2654
2655    default:
2656      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2657      break;
2658  }
2659}
2660
2661void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
2662  LocationSummary* locations = add->GetLocations();
2663  Location first = locations->InAt(0);
2664  Location second = locations->InAt(1);
2665  Location out = locations->Out();
2666
2667  switch (add->GetResultType()) {
2668    case Primitive::kPrimInt: {
2669      if (second.IsRegister()) {
2670        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2671          __ addl(out.AsRegister<Register>(), second.AsRegister<Register>());
2672        } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
2673          __ addl(out.AsRegister<Register>(), first.AsRegister<Register>());
2674        } else {
2675          __ leal(out.AsRegister<Register>(), Address(
2676              first.AsRegister<Register>(), second.AsRegister<Register>(), TIMES_1, 0));
2677          }
2678      } else if (second.IsConstant()) {
2679        int32_t value = second.GetConstant()->AsIntConstant()->GetValue();
2680        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2681          __ addl(out.AsRegister<Register>(), Immediate(value));
2682        } else {
2683          __ leal(out.AsRegister<Register>(), Address(first.AsRegister<Register>(), value));
2684        }
2685      } else {
2686        DCHECK(first.Equals(locations->Out()));
2687        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2688      }
2689      break;
2690    }
2691
2692    case Primitive::kPrimLong: {
2693      if (second.IsRegisterPair()) {
2694        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2695        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2696      } else if (second.IsDoubleStackSlot()) {
2697        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2698        __ adcl(first.AsRegisterPairHigh<Register>(),
2699                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2700      } else {
2701        DCHECK(second.IsConstant()) << second;
2702        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2703        __ addl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2704        __ adcl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2705      }
2706      break;
2707    }
2708
2709    case Primitive::kPrimFloat: {
2710      if (second.IsFpuRegister()) {
2711        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2712      } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2713        HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2714        DCHECK(!const_area->NeedsMaterialization());
2715        __ addss(first.AsFpuRegister<XmmRegister>(),
2716                 codegen_->LiteralFloatAddress(
2717                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2718                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2719      } else {
2720        DCHECK(second.IsStackSlot());
2721        __ addss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2722      }
2723      break;
2724    }
2725
2726    case Primitive::kPrimDouble: {
2727      if (second.IsFpuRegister()) {
2728        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2729      } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2730        HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2731        DCHECK(!const_area->NeedsMaterialization());
2732        __ addsd(first.AsFpuRegister<XmmRegister>(),
2733                 codegen_->LiteralDoubleAddress(
2734                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2735                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2736      } else {
2737        DCHECK(second.IsDoubleStackSlot());
2738        __ addsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2739      }
2740      break;
2741    }
2742
2743    default:
2744      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2745  }
2746}
2747
2748void LocationsBuilderX86::VisitSub(HSub* sub) {
2749  LocationSummary* locations =
2750      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2751  switch (sub->GetResultType()) {
2752    case Primitive::kPrimInt:
2753    case Primitive::kPrimLong: {
2754      locations->SetInAt(0, Location::RequiresRegister());
2755      locations->SetInAt(1, Location::Any());
2756      locations->SetOut(Location::SameAsFirstInput());
2757      break;
2758    }
2759    case Primitive::kPrimFloat:
2760    case Primitive::kPrimDouble: {
2761      locations->SetInAt(0, Location::RequiresFpuRegister());
2762      locations->SetInAt(1, Location::Any());
2763      locations->SetOut(Location::SameAsFirstInput());
2764      break;
2765    }
2766
2767    default:
2768      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2769  }
2770}
2771
2772void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
2773  LocationSummary* locations = sub->GetLocations();
2774  Location first = locations->InAt(0);
2775  Location second = locations->InAt(1);
2776  DCHECK(first.Equals(locations->Out()));
2777  switch (sub->GetResultType()) {
2778    case Primitive::kPrimInt: {
2779      if (second.IsRegister()) {
2780        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
2781      } else if (second.IsConstant()) {
2782        __ subl(first.AsRegister<Register>(),
2783                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2784      } else {
2785        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2786      }
2787      break;
2788    }
2789
2790    case Primitive::kPrimLong: {
2791      if (second.IsRegisterPair()) {
2792        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2793        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2794      } else if (second.IsDoubleStackSlot()) {
2795        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2796        __ sbbl(first.AsRegisterPairHigh<Register>(),
2797                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2798      } else {
2799        DCHECK(second.IsConstant()) << second;
2800        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2801        __ subl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2802        __ sbbl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2803      }
2804      break;
2805    }
2806
2807    case Primitive::kPrimFloat: {
2808      if (second.IsFpuRegister()) {
2809        __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2810      } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2811        HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2812        DCHECK(!const_area->NeedsMaterialization());
2813        __ subss(first.AsFpuRegister<XmmRegister>(),
2814                 codegen_->LiteralFloatAddress(
2815                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2816                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2817      } else {
2818        DCHECK(second.IsStackSlot());
2819        __ subss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2820      }
2821      break;
2822    }
2823
2824    case Primitive::kPrimDouble: {
2825      if (second.IsFpuRegister()) {
2826        __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2827      } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2828        HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2829        DCHECK(!const_area->NeedsMaterialization());
2830        __ subsd(first.AsFpuRegister<XmmRegister>(),
2831                 codegen_->LiteralDoubleAddress(
2832                     const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2833                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2834      } else {
2835        DCHECK(second.IsDoubleStackSlot());
2836        __ subsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2837      }
2838      break;
2839    }
2840
2841    default:
2842      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2843  }
2844}
2845
2846void LocationsBuilderX86::VisitMul(HMul* mul) {
2847  LocationSummary* locations =
2848      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2849  switch (mul->GetResultType()) {
2850    case Primitive::kPrimInt:
2851      locations->SetInAt(0, Location::RequiresRegister());
2852      locations->SetInAt(1, Location::Any());
2853      if (mul->InputAt(1)->IsIntConstant()) {
2854        // Can use 3 operand multiply.
2855        locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2856      } else {
2857        locations->SetOut(Location::SameAsFirstInput());
2858      }
2859      break;
2860    case Primitive::kPrimLong: {
2861      locations->SetInAt(0, Location::RequiresRegister());
2862      locations->SetInAt(1, Location::Any());
2863      locations->SetOut(Location::SameAsFirstInput());
2864      // Needed for imul on 32bits with 64bits output.
2865      locations->AddTemp(Location::RegisterLocation(EAX));
2866      locations->AddTemp(Location::RegisterLocation(EDX));
2867      break;
2868    }
2869    case Primitive::kPrimFloat:
2870    case Primitive::kPrimDouble: {
2871      locations->SetInAt(0, Location::RequiresFpuRegister());
2872      locations->SetInAt(1, Location::Any());
2873      locations->SetOut(Location::SameAsFirstInput());
2874      break;
2875    }
2876
2877    default:
2878      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2879  }
2880}
2881
2882void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2883  LocationSummary* locations = mul->GetLocations();
2884  Location first = locations->InAt(0);
2885  Location second = locations->InAt(1);
2886  Location out = locations->Out();
2887
2888  switch (mul->GetResultType()) {
2889    case Primitive::kPrimInt:
2890      // The constant may have ended up in a register, so test explicitly to avoid
2891      // problems where the output may not be the same as the first operand.
2892      if (mul->InputAt(1)->IsIntConstant()) {
2893        Immediate imm(mul->InputAt(1)->AsIntConstant()->GetValue());
2894        __ imull(out.AsRegister<Register>(), first.AsRegister<Register>(), imm);
2895      } else if (second.IsRegister()) {
2896        DCHECK(first.Equals(out));
2897        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
2898      } else {
2899        DCHECK(second.IsStackSlot());
2900        DCHECK(first.Equals(out));
2901        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2902      }
2903      break;
2904
2905    case Primitive::kPrimLong: {
2906      Register in1_hi = first.AsRegisterPairHigh<Register>();
2907      Register in1_lo = first.AsRegisterPairLow<Register>();
2908      Register eax = locations->GetTemp(0).AsRegister<Register>();
2909      Register edx = locations->GetTemp(1).AsRegister<Register>();
2910
2911      DCHECK_EQ(EAX, eax);
2912      DCHECK_EQ(EDX, edx);
2913
2914      // input: in1 - 64 bits, in2 - 64 bits.
2915      // output: in1
2916      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2917      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2918      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
2919      if (second.IsConstant()) {
2920        DCHECK(second.GetConstant()->IsLongConstant());
2921
2922        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2923        int32_t low_value = Low32Bits(value);
2924        int32_t high_value = High32Bits(value);
2925        Immediate low(low_value);
2926        Immediate high(high_value);
2927
2928        __ movl(eax, high);
2929        // eax <- in1.lo * in2.hi
2930        __ imull(eax, in1_lo);
2931        // in1.hi <- in1.hi * in2.lo
2932        __ imull(in1_hi, low);
2933        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2934        __ addl(in1_hi, eax);
2935        // move in2_lo to eax to prepare for double precision
2936        __ movl(eax, low);
2937        // edx:eax <- in1.lo * in2.lo
2938        __ mull(in1_lo);
2939        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2940        __ addl(in1_hi, edx);
2941        // in1.lo <- (in1.lo * in2.lo)[31:0];
2942        __ movl(in1_lo, eax);
2943      } else if (second.IsRegisterPair()) {
2944        Register in2_hi = second.AsRegisterPairHigh<Register>();
2945        Register in2_lo = second.AsRegisterPairLow<Register>();
2946
2947        __ movl(eax, in2_hi);
2948        // eax <- in1.lo * in2.hi
2949        __ imull(eax, in1_lo);
2950        // in1.hi <- in1.hi * in2.lo
2951        __ imull(in1_hi, in2_lo);
2952        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2953        __ addl(in1_hi, eax);
2954        // move in1_lo to eax to prepare for double precision
2955        __ movl(eax, in1_lo);
2956        // edx:eax <- in1.lo * in2.lo
2957        __ mull(in2_lo);
2958        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2959        __ addl(in1_hi, edx);
2960        // in1.lo <- (in1.lo * in2.lo)[31:0];
2961        __ movl(in1_lo, eax);
2962      } else {
2963        DCHECK(second.IsDoubleStackSlot()) << second;
2964        Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
2965        Address in2_lo(ESP, second.GetStackIndex());
2966
2967        __ movl(eax, in2_hi);
2968        // eax <- in1.lo * in2.hi
2969        __ imull(eax, in1_lo);
2970        // in1.hi <- in1.hi * in2.lo
2971        __ imull(in1_hi, in2_lo);
2972        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2973        __ addl(in1_hi, eax);
2974        // move in1_lo to eax to prepare for double precision
2975        __ movl(eax, in1_lo);
2976        // edx:eax <- in1.lo * in2.lo
2977        __ mull(in2_lo);
2978        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2979        __ addl(in1_hi, edx);
2980        // in1.lo <- (in1.lo * in2.lo)[31:0];
2981        __ movl(in1_lo, eax);
2982      }
2983
2984      break;
2985    }
2986
2987    case Primitive::kPrimFloat: {
2988      DCHECK(first.Equals(locations->Out()));
2989      if (second.IsFpuRegister()) {
2990        __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2991      } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
2992        HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
2993        DCHECK(!const_area->NeedsMaterialization());
2994        __ mulss(first.AsFpuRegister<XmmRegister>(),
2995                 codegen_->LiteralFloatAddress(
2996                     const_area->GetConstant()->AsFloatConstant()->GetValue(),
2997                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2998      } else {
2999        DCHECK(second.IsStackSlot());
3000        __ mulss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3001      }
3002      break;
3003    }
3004
3005    case Primitive::kPrimDouble: {
3006      DCHECK(first.Equals(locations->Out()));
3007      if (second.IsFpuRegister()) {
3008        __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3009      } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
3010        HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
3011        DCHECK(!const_area->NeedsMaterialization());
3012        __ mulsd(first.AsFpuRegister<XmmRegister>(),
3013                 codegen_->LiteralDoubleAddress(
3014                     const_area->GetConstant()->AsDoubleConstant()->GetValue(),
3015                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3016      } else {
3017        DCHECK(second.IsDoubleStackSlot());
3018        __ mulsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3019      }
3020      break;
3021    }
3022
3023    default:
3024      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
3025  }
3026}
3027
3028void InstructionCodeGeneratorX86::PushOntoFPStack(Location source,
3029                                                  uint32_t temp_offset,
3030                                                  uint32_t stack_adjustment,
3031                                                  bool is_fp,
3032                                                  bool is_wide) {
3033  if (source.IsStackSlot()) {
3034    DCHECK(!is_wide);
3035    if (is_fp) {
3036      __ flds(Address(ESP, source.GetStackIndex() + stack_adjustment));
3037    } else {
3038      __ filds(Address(ESP, source.GetStackIndex() + stack_adjustment));
3039    }
3040  } else if (source.IsDoubleStackSlot()) {
3041    DCHECK(is_wide);
3042    if (is_fp) {
3043      __ fldl(Address(ESP, source.GetStackIndex() + stack_adjustment));
3044    } else {
3045      __ fildl(Address(ESP, source.GetStackIndex() + stack_adjustment));
3046    }
3047  } else {
3048    // Write the value to the temporary location on the stack and load to FP stack.
3049    if (!is_wide) {
3050      Location stack_temp = Location::StackSlot(temp_offset);
3051      codegen_->Move32(stack_temp, source);
3052      if (is_fp) {
3053        __ flds(Address(ESP, temp_offset));
3054      } else {
3055        __ filds(Address(ESP, temp_offset));
3056      }
3057    } else {
3058      Location stack_temp = Location::DoubleStackSlot(temp_offset);
3059      codegen_->Move64(stack_temp, source);
3060      if (is_fp) {
3061        __ fldl(Address(ESP, temp_offset));
3062      } else {
3063        __ fildl(Address(ESP, temp_offset));
3064      }
3065    }
3066  }
3067}
3068
3069void InstructionCodeGeneratorX86::GenerateRemFP(HRem *rem) {
3070  Primitive::Type type = rem->GetResultType();
3071  bool is_float = type == Primitive::kPrimFloat;
3072  size_t elem_size = Primitive::ComponentSize(type);
3073  LocationSummary* locations = rem->GetLocations();
3074  Location first = locations->InAt(0);
3075  Location second = locations->InAt(1);
3076  Location out = locations->Out();
3077
3078  // Create stack space for 2 elements.
3079  // TODO: enhance register allocator to ask for stack temporaries.
3080  __ subl(ESP, Immediate(2 * elem_size));
3081
3082  // Load the values to the FP stack in reverse order, using temporaries if needed.
3083  const bool is_wide = !is_float;
3084  PushOntoFPStack(second, elem_size, 2 * elem_size, /* is_fp */ true, is_wide);
3085  PushOntoFPStack(first, 0, 2 * elem_size, /* is_fp */ true, is_wide);
3086
3087  // Loop doing FPREM until we stabilize.
3088  NearLabel retry;
3089  __ Bind(&retry);
3090  __ fprem();
3091
3092  // Move FP status to AX.
3093  __ fstsw();
3094
3095  // And see if the argument reduction is complete. This is signaled by the
3096  // C2 FPU flag bit set to 0.
3097  __ andl(EAX, Immediate(kC2ConditionMask));
3098  __ j(kNotEqual, &retry);
3099
3100  // We have settled on the final value. Retrieve it into an XMM register.
3101  // Store FP top of stack to real stack.
3102  if (is_float) {
3103    __ fsts(Address(ESP, 0));
3104  } else {
3105    __ fstl(Address(ESP, 0));
3106  }
3107
3108  // Pop the 2 items from the FP stack.
3109  __ fucompp();
3110
3111  // Load the value from the stack into an XMM register.
3112  DCHECK(out.IsFpuRegister()) << out;
3113  if (is_float) {
3114    __ movss(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
3115  } else {
3116    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
3117  }
3118
3119  // And remove the temporary stack space we allocated.
3120  __ addl(ESP, Immediate(2 * elem_size));
3121}
3122
3123
3124void InstructionCodeGeneratorX86::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
3125  DCHECK(instruction->IsDiv() || instruction->IsRem());
3126
3127  LocationSummary* locations = instruction->GetLocations();
3128  DCHECK(locations->InAt(1).IsConstant());
3129  DCHECK(locations->InAt(1).GetConstant()->IsIntConstant());
3130
3131  Register out_register = locations->Out().AsRegister<Register>();
3132  Register input_register = locations->InAt(0).AsRegister<Register>();
3133  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3134
3135  DCHECK(imm == 1 || imm == -1);
3136
3137  if (instruction->IsRem()) {
3138    __ xorl(out_register, out_register);
3139  } else {
3140    __ movl(out_register, input_register);
3141    if (imm == -1) {
3142      __ negl(out_register);
3143    }
3144  }
3145}
3146
3147
3148void InstructionCodeGeneratorX86::DivByPowerOfTwo(HDiv* instruction) {
3149  LocationSummary* locations = instruction->GetLocations();
3150
3151  Register out_register = locations->Out().AsRegister<Register>();
3152  Register input_register = locations->InAt(0).AsRegister<Register>();
3153  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3154
3155  DCHECK(IsPowerOfTwo(std::abs(imm)));
3156  Register num = locations->GetTemp(0).AsRegister<Register>();
3157
3158  __ leal(num, Address(input_register, std::abs(imm) - 1));
3159  __ testl(input_register, input_register);
3160  __ cmovl(kGreaterEqual, num, input_register);
3161  int shift = CTZ(imm);
3162  __ sarl(num, Immediate(shift));
3163
3164  if (imm < 0) {
3165    __ negl(num);
3166  }
3167
3168  __ movl(out_register, num);
3169}
3170
3171void InstructionCodeGeneratorX86::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
3172  DCHECK(instruction->IsDiv() || instruction->IsRem());
3173
3174  LocationSummary* locations = instruction->GetLocations();
3175  int imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3176
3177  Register eax = locations->InAt(0).AsRegister<Register>();
3178  Register out = locations->Out().AsRegister<Register>();
3179  Register num;
3180  Register edx;
3181
3182  if (instruction->IsDiv()) {
3183    edx = locations->GetTemp(0).AsRegister<Register>();
3184    num = locations->GetTemp(1).AsRegister<Register>();
3185  } else {
3186    edx = locations->Out().AsRegister<Register>();
3187    num = locations->GetTemp(0).AsRegister<Register>();
3188  }
3189
3190  DCHECK_EQ(EAX, eax);
3191  DCHECK_EQ(EDX, edx);
3192  if (instruction->IsDiv()) {
3193    DCHECK_EQ(EAX, out);
3194  } else {
3195    DCHECK_EQ(EDX, out);
3196  }
3197
3198  int64_t magic;
3199  int shift;
3200  CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
3201
3202  NearLabel ndiv;
3203  NearLabel end;
3204  // If numerator is 0, the result is 0, no computation needed.
3205  __ testl(eax, eax);
3206  __ j(kNotEqual, &ndiv);
3207
3208  __ xorl(out, out);
3209  __ jmp(&end);
3210
3211  __ Bind(&ndiv);
3212
3213  // Save the numerator.
3214  __ movl(num, eax);
3215
3216  // EAX = magic
3217  __ movl(eax, Immediate(magic));
3218
3219  // EDX:EAX = magic * numerator
3220  __ imull(num);
3221
3222  if (imm > 0 && magic < 0) {
3223    // EDX += num
3224    __ addl(edx, num);
3225  } else if (imm < 0 && magic > 0) {
3226    __ subl(edx, num);
3227  }
3228
3229  // Shift if needed.
3230  if (shift != 0) {
3231    __ sarl(edx, Immediate(shift));
3232  }
3233
3234  // EDX += 1 if EDX < 0
3235  __ movl(eax, edx);
3236  __ shrl(edx, Immediate(31));
3237  __ addl(edx, eax);
3238
3239  if (instruction->IsRem()) {
3240    __ movl(eax, num);
3241    __ imull(edx, Immediate(imm));
3242    __ subl(eax, edx);
3243    __ movl(edx, eax);
3244  } else {
3245    __ movl(eax, edx);
3246  }
3247  __ Bind(&end);
3248}
3249
3250void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
3251  DCHECK(instruction->IsDiv() || instruction->IsRem());
3252
3253  LocationSummary* locations = instruction->GetLocations();
3254  Location out = locations->Out();
3255  Location first = locations->InAt(0);
3256  Location second = locations->InAt(1);
3257  bool is_div = instruction->IsDiv();
3258
3259  switch (instruction->GetResultType()) {
3260    case Primitive::kPrimInt: {
3261      DCHECK_EQ(EAX, first.AsRegister<Register>());
3262      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
3263
3264      if (second.IsConstant()) {
3265        int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
3266
3267        if (imm == 0) {
3268          // Do not generate anything for 0. DivZeroCheck would forbid any generated code.
3269        } else if (imm == 1 || imm == -1) {
3270          DivRemOneOrMinusOne(instruction);
3271        } else if (is_div && IsPowerOfTwo(std::abs(imm))) {
3272          DivByPowerOfTwo(instruction->AsDiv());
3273        } else {
3274          DCHECK(imm <= -2 || imm >= 2);
3275          GenerateDivRemWithAnyConstant(instruction);
3276        }
3277      } else {
3278        SlowPathCode* slow_path =
3279          new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(out.AsRegister<Register>(),
3280              is_div);
3281        codegen_->AddSlowPath(slow_path);
3282
3283        Register second_reg = second.AsRegister<Register>();
3284        // 0x80000000/-1 triggers an arithmetic exception!
3285        // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
3286        // it's safe to just use negl instead of more complex comparisons.
3287
3288        __ cmpl(second_reg, Immediate(-1));
3289        __ j(kEqual, slow_path->GetEntryLabel());
3290
3291        // edx:eax <- sign-extended of eax
3292        __ cdq();
3293        // eax = quotient, edx = remainder
3294        __ idivl(second_reg);
3295        __ Bind(slow_path->GetExitLabel());
3296      }
3297      break;
3298    }
3299
3300    case Primitive::kPrimLong: {
3301      InvokeRuntimeCallingConvention calling_convention;
3302      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
3303      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
3304      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
3305      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
3306      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
3307      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
3308
3309      if (is_div) {
3310        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLdiv),
3311                                instruction,
3312                                instruction->GetDexPc(),
3313                                nullptr);
3314        CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>();
3315      } else {
3316        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLmod),
3317                                instruction,
3318                                instruction->GetDexPc(),
3319                                nullptr);
3320        CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>();
3321      }
3322      break;
3323    }
3324
3325    default:
3326      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
3327  }
3328}
3329
3330void LocationsBuilderX86::VisitDiv(HDiv* div) {
3331  LocationSummary::CallKind call_kind = (div->GetResultType() == Primitive::kPrimLong)
3332      ? LocationSummary::kCall
3333      : LocationSummary::kNoCall;
3334  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
3335
3336  switch (div->GetResultType()) {
3337    case Primitive::kPrimInt: {
3338      locations->SetInAt(0, Location::RegisterLocation(EAX));
3339      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
3340      locations->SetOut(Location::SameAsFirstInput());
3341      // Intel uses edx:eax as the dividend.
3342      locations->AddTemp(Location::RegisterLocation(EDX));
3343      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
3344      // which enforces results to be in EAX and EDX, things are simpler if we use EAX also as
3345      // output and request another temp.
3346      if (div->InputAt(1)->IsIntConstant()) {
3347        locations->AddTemp(Location::RequiresRegister());
3348      }
3349      break;
3350    }
3351    case Primitive::kPrimLong: {
3352      InvokeRuntimeCallingConvention calling_convention;
3353      locations->SetInAt(0, Location::RegisterPairLocation(
3354          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
3355      locations->SetInAt(1, Location::RegisterPairLocation(
3356          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
3357      // Runtime helper puts the result in EAX, EDX.
3358      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
3359      break;
3360    }
3361    case Primitive::kPrimFloat:
3362    case Primitive::kPrimDouble: {
3363      locations->SetInAt(0, Location::RequiresFpuRegister());
3364      locations->SetInAt(1, Location::Any());
3365      locations->SetOut(Location::SameAsFirstInput());
3366      break;
3367    }
3368
3369    default:
3370      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
3371  }
3372}
3373
3374void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
3375  LocationSummary* locations = div->GetLocations();
3376  Location first = locations->InAt(0);
3377  Location second = locations->InAt(1);
3378
3379  switch (div->GetResultType()) {
3380    case Primitive::kPrimInt:
3381    case Primitive::kPrimLong: {
3382      GenerateDivRemIntegral(div);
3383      break;
3384    }
3385
3386    case Primitive::kPrimFloat: {
3387      if (second.IsFpuRegister()) {
3388        __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3389      } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3390        HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3391        DCHECK(!const_area->NeedsMaterialization());
3392        __ divss(first.AsFpuRegister<XmmRegister>(),
3393                 codegen_->LiteralFloatAddress(
3394                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
3395                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3396      } else {
3397        DCHECK(second.IsStackSlot());
3398        __ divss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3399      }
3400      break;
3401    }
3402
3403    case Primitive::kPrimDouble: {
3404      if (second.IsFpuRegister()) {
3405        __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3406      } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3407        HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3408        DCHECK(!const_area->NeedsMaterialization());
3409        __ divsd(first.AsFpuRegister<XmmRegister>(),
3410                 codegen_->LiteralDoubleAddress(
3411                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
3412                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3413      } else {
3414        DCHECK(second.IsDoubleStackSlot());
3415        __ divsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3416      }
3417      break;
3418    }
3419
3420    default:
3421      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
3422  }
3423}
3424
3425void LocationsBuilderX86::VisitRem(HRem* rem) {
3426  Primitive::Type type = rem->GetResultType();
3427
3428  LocationSummary::CallKind call_kind = (rem->GetResultType() == Primitive::kPrimLong)
3429      ? LocationSummary::kCall
3430      : LocationSummary::kNoCall;
3431  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
3432
3433  switch (type) {
3434    case Primitive::kPrimInt: {
3435      locations->SetInAt(0, Location::RegisterLocation(EAX));
3436      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
3437      locations->SetOut(Location::RegisterLocation(EDX));
3438      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
3439      // which enforces results to be in EAX and EDX, things are simpler if we use EDX also as
3440      // output and request another temp.
3441      if (rem->InputAt(1)->IsIntConstant()) {
3442        locations->AddTemp(Location::RequiresRegister());
3443      }
3444      break;
3445    }
3446    case Primitive::kPrimLong: {
3447      InvokeRuntimeCallingConvention calling_convention;
3448      locations->SetInAt(0, Location::RegisterPairLocation(
3449          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
3450      locations->SetInAt(1, Location::RegisterPairLocation(
3451          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
3452      // Runtime helper puts the result in EAX, EDX.
3453      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
3454      break;
3455    }
3456    case Primitive::kPrimDouble:
3457    case Primitive::kPrimFloat: {
3458      locations->SetInAt(0, Location::Any());
3459      locations->SetInAt(1, Location::Any());
3460      locations->SetOut(Location::RequiresFpuRegister());
3461      locations->AddTemp(Location::RegisterLocation(EAX));
3462      break;
3463    }
3464
3465    default:
3466      LOG(FATAL) << "Unexpected rem type " << type;
3467  }
3468}
3469
3470void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
3471  Primitive::Type type = rem->GetResultType();
3472  switch (type) {
3473    case Primitive::kPrimInt:
3474    case Primitive::kPrimLong: {
3475      GenerateDivRemIntegral(rem);
3476      break;
3477    }
3478    case Primitive::kPrimFloat:
3479    case Primitive::kPrimDouble: {
3480      GenerateRemFP(rem);
3481      break;
3482    }
3483    default:
3484      LOG(FATAL) << "Unexpected rem type " << type;
3485  }
3486}
3487
3488void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3489  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
3490      ? LocationSummary::kCallOnSlowPath
3491      : LocationSummary::kNoCall;
3492  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3493  switch (instruction->GetType()) {
3494    case Primitive::kPrimByte:
3495    case Primitive::kPrimChar:
3496    case Primitive::kPrimShort:
3497    case Primitive::kPrimInt: {
3498      locations->SetInAt(0, Location::Any());
3499      break;
3500    }
3501    case Primitive::kPrimLong: {
3502      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3503      if (!instruction->IsConstant()) {
3504        locations->AddTemp(Location::RequiresRegister());
3505      }
3506      break;
3507    }
3508    default:
3509      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
3510  }
3511  if (instruction->HasUses()) {
3512    locations->SetOut(Location::SameAsFirstInput());
3513  }
3514}
3515
3516void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3517  SlowPathCode* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
3518  codegen_->AddSlowPath(slow_path);
3519
3520  LocationSummary* locations = instruction->GetLocations();
3521  Location value = locations->InAt(0);
3522
3523  switch (instruction->GetType()) {
3524    case Primitive::kPrimByte:
3525    case Primitive::kPrimChar:
3526    case Primitive::kPrimShort:
3527    case Primitive::kPrimInt: {
3528      if (value.IsRegister()) {
3529        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
3530        __ j(kEqual, slow_path->GetEntryLabel());
3531      } else if (value.IsStackSlot()) {
3532        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
3533        __ j(kEqual, slow_path->GetEntryLabel());
3534      } else {
3535        DCHECK(value.IsConstant()) << value;
3536        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
3537        __ jmp(slow_path->GetEntryLabel());
3538        }
3539      }
3540      break;
3541    }
3542    case Primitive::kPrimLong: {
3543      if (value.IsRegisterPair()) {
3544        Register temp = locations->GetTemp(0).AsRegister<Register>();
3545        __ movl(temp, value.AsRegisterPairLow<Register>());
3546        __ orl(temp, value.AsRegisterPairHigh<Register>());
3547        __ j(kEqual, slow_path->GetEntryLabel());
3548      } else {
3549        DCHECK(value.IsConstant()) << value;
3550        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
3551          __ jmp(slow_path->GetEntryLabel());
3552        }
3553      }
3554      break;
3555    }
3556    default:
3557      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
3558  }
3559}
3560
3561void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
3562  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3563
3564  LocationSummary* locations =
3565      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
3566
3567  switch (op->GetResultType()) {
3568    case Primitive::kPrimInt:
3569    case Primitive::kPrimLong: {
3570      // Can't have Location::Any() and output SameAsFirstInput()
3571      locations->SetInAt(0, Location::RequiresRegister());
3572      // The shift count needs to be in CL or a constant.
3573      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
3574      locations->SetOut(Location::SameAsFirstInput());
3575      break;
3576    }
3577    default:
3578      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3579  }
3580}
3581
3582void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
3583  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3584
3585  LocationSummary* locations = op->GetLocations();
3586  Location first = locations->InAt(0);
3587  Location second = locations->InAt(1);
3588  DCHECK(first.Equals(locations->Out()));
3589
3590  switch (op->GetResultType()) {
3591    case Primitive::kPrimInt: {
3592      DCHECK(first.IsRegister());
3593      Register first_reg = first.AsRegister<Register>();
3594      if (second.IsRegister()) {
3595        Register second_reg = second.AsRegister<Register>();
3596        DCHECK_EQ(ECX, second_reg);
3597        if (op->IsShl()) {
3598          __ shll(first_reg, second_reg);
3599        } else if (op->IsShr()) {
3600          __ sarl(first_reg, second_reg);
3601        } else {
3602          __ shrl(first_reg, second_reg);
3603        }
3604      } else {
3605        int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue;
3606        if (shift == 0) {
3607          return;
3608        }
3609        Immediate imm(shift);
3610        if (op->IsShl()) {
3611          __ shll(first_reg, imm);
3612        } else if (op->IsShr()) {
3613          __ sarl(first_reg, imm);
3614        } else {
3615          __ shrl(first_reg, imm);
3616        }
3617      }
3618      break;
3619    }
3620    case Primitive::kPrimLong: {
3621      if (second.IsRegister()) {
3622        Register second_reg = second.AsRegister<Register>();
3623        DCHECK_EQ(ECX, second_reg);
3624        if (op->IsShl()) {
3625          GenerateShlLong(first, second_reg);
3626        } else if (op->IsShr()) {
3627          GenerateShrLong(first, second_reg);
3628        } else {
3629          GenerateUShrLong(first, second_reg);
3630        }
3631      } else {
3632        // Shift by a constant.
3633        int shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue;
3634        // Nothing to do if the shift is 0, as the input is already the output.
3635        if (shift != 0) {
3636          if (op->IsShl()) {
3637            GenerateShlLong(first, shift);
3638          } else if (op->IsShr()) {
3639            GenerateShrLong(first, shift);
3640          } else {
3641            GenerateUShrLong(first, shift);
3642          }
3643        }
3644      }
3645      break;
3646    }
3647    default:
3648      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3649  }
3650}
3651
3652void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, int shift) {
3653  Register low = loc.AsRegisterPairLow<Register>();
3654  Register high = loc.AsRegisterPairHigh<Register>();
3655  if (shift == 1) {
3656    // This is just an addition.
3657    __ addl(low, low);
3658    __ adcl(high, high);
3659  } else if (shift == 32) {
3660    // Shift by 32 is easy. High gets low, and low gets 0.
3661    codegen_->EmitParallelMoves(
3662        loc.ToLow(),
3663        loc.ToHigh(),
3664        Primitive::kPrimInt,
3665        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3666        loc.ToLow(),
3667        Primitive::kPrimInt);
3668  } else if (shift > 32) {
3669    // Low part becomes 0.  High part is low part << (shift-32).
3670    __ movl(high, low);
3671    __ shll(high, Immediate(shift - 32));
3672    __ xorl(low, low);
3673  } else {
3674    // Between 1 and 31.
3675    __ shld(high, low, Immediate(shift));
3676    __ shll(low, Immediate(shift));
3677  }
3678}
3679
3680void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
3681  NearLabel done;
3682  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
3683  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
3684  __ testl(shifter, Immediate(32));
3685  __ j(kEqual, &done);
3686  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
3687  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
3688  __ Bind(&done);
3689}
3690
3691void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, int shift) {
3692  Register low = loc.AsRegisterPairLow<Register>();
3693  Register high = loc.AsRegisterPairHigh<Register>();
3694  if (shift == 32) {
3695    // Need to copy the sign.
3696    DCHECK_NE(low, high);
3697    __ movl(low, high);
3698    __ sarl(high, Immediate(31));
3699  } else if (shift > 32) {
3700    DCHECK_NE(low, high);
3701    // High part becomes sign. Low part is shifted by shift - 32.
3702    __ movl(low, high);
3703    __ sarl(high, Immediate(31));
3704    __ sarl(low, Immediate(shift - 32));
3705  } else {
3706    // Between 1 and 31.
3707    __ shrd(low, high, Immediate(shift));
3708    __ sarl(high, Immediate(shift));
3709  }
3710}
3711
3712void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
3713  NearLabel done;
3714  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3715  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
3716  __ testl(shifter, Immediate(32));
3717  __ j(kEqual, &done);
3718  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3719  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
3720  __ Bind(&done);
3721}
3722
3723void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, int shift) {
3724  Register low = loc.AsRegisterPairLow<Register>();
3725  Register high = loc.AsRegisterPairHigh<Register>();
3726  if (shift == 32) {
3727    // Shift by 32 is easy. Low gets high, and high gets 0.
3728    codegen_->EmitParallelMoves(
3729        loc.ToHigh(),
3730        loc.ToLow(),
3731        Primitive::kPrimInt,
3732        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3733        loc.ToHigh(),
3734        Primitive::kPrimInt);
3735  } else if (shift > 32) {
3736    // Low part is high >> (shift - 32). High part becomes 0.
3737    __ movl(low, high);
3738    __ shrl(low, Immediate(shift - 32));
3739    __ xorl(high, high);
3740  } else {
3741    // Between 1 and 31.
3742    __ shrd(low, high, Immediate(shift));
3743    __ shrl(high, Immediate(shift));
3744  }
3745}
3746
3747void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
3748  NearLabel done;
3749  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3750  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
3751  __ testl(shifter, Immediate(32));
3752  __ j(kEqual, &done);
3753  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3754  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
3755  __ Bind(&done);
3756}
3757
3758void LocationsBuilderX86::VisitShl(HShl* shl) {
3759  HandleShift(shl);
3760}
3761
3762void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
3763  HandleShift(shl);
3764}
3765
3766void LocationsBuilderX86::VisitShr(HShr* shr) {
3767  HandleShift(shr);
3768}
3769
3770void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
3771  HandleShift(shr);
3772}
3773
3774void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
3775  HandleShift(ushr);
3776}
3777
3778void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
3779  HandleShift(ushr);
3780}
3781
3782void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
3783  LocationSummary* locations =
3784      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3785  locations->SetOut(Location::RegisterLocation(EAX));
3786  InvokeRuntimeCallingConvention calling_convention;
3787  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3788  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3789}
3790
3791void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
3792  // Note: if heap poisoning is enabled, the entry point takes cares
3793  // of poisoning the reference.
3794  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3795                          instruction,
3796                          instruction->GetDexPc(),
3797                          nullptr);
3798  CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
3799  DCHECK(!codegen_->IsLeafMethod());
3800}
3801
3802void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
3803  LocationSummary* locations =
3804      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3805  locations->SetOut(Location::RegisterLocation(EAX));
3806  InvokeRuntimeCallingConvention calling_convention;
3807  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3808  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3809  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3810}
3811
3812void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
3813  InvokeRuntimeCallingConvention calling_convention;
3814  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
3815  // Note: if heap poisoning is enabled, the entry point takes cares
3816  // of poisoning the reference.
3817  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3818                          instruction,
3819                          instruction->GetDexPc(),
3820                          nullptr);
3821  CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>();
3822  DCHECK(!codegen_->IsLeafMethod());
3823}
3824
3825void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
3826  LocationSummary* locations =
3827      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3828  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3829  if (location.IsStackSlot()) {
3830    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3831  } else if (location.IsDoubleStackSlot()) {
3832    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3833  }
3834  locations->SetOut(location);
3835}
3836
3837void InstructionCodeGeneratorX86::VisitParameterValue(
3838    HParameterValue* instruction ATTRIBUTE_UNUSED) {
3839}
3840
3841void LocationsBuilderX86::VisitCurrentMethod(HCurrentMethod* instruction) {
3842  LocationSummary* locations =
3843      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3844  locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument));
3845}
3846
3847void InstructionCodeGeneratorX86::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
3848}
3849
3850void LocationsBuilderX86::VisitNot(HNot* not_) {
3851  LocationSummary* locations =
3852      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
3853  locations->SetInAt(0, Location::RequiresRegister());
3854  locations->SetOut(Location::SameAsFirstInput());
3855}
3856
3857void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
3858  LocationSummary* locations = not_->GetLocations();
3859  Location in = locations->InAt(0);
3860  Location out = locations->Out();
3861  DCHECK(in.Equals(out));
3862  switch (not_->GetResultType()) {
3863    case Primitive::kPrimInt:
3864      __ notl(out.AsRegister<Register>());
3865      break;
3866
3867    case Primitive::kPrimLong:
3868      __ notl(out.AsRegisterPairLow<Register>());
3869      __ notl(out.AsRegisterPairHigh<Register>());
3870      break;
3871
3872    default:
3873      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
3874  }
3875}
3876
3877void LocationsBuilderX86::VisitBooleanNot(HBooleanNot* bool_not) {
3878  LocationSummary* locations =
3879      new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
3880  locations->SetInAt(0, Location::RequiresRegister());
3881  locations->SetOut(Location::SameAsFirstInput());
3882}
3883
3884void InstructionCodeGeneratorX86::VisitBooleanNot(HBooleanNot* bool_not) {
3885  LocationSummary* locations = bool_not->GetLocations();
3886  Location in = locations->InAt(0);
3887  Location out = locations->Out();
3888  DCHECK(in.Equals(out));
3889  __ xorl(out.AsRegister<Register>(), Immediate(1));
3890}
3891
3892void LocationsBuilderX86::VisitCompare(HCompare* compare) {
3893  LocationSummary* locations =
3894      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
3895  switch (compare->InputAt(0)->GetType()) {
3896    case Primitive::kPrimLong: {
3897      locations->SetInAt(0, Location::RequiresRegister());
3898      locations->SetInAt(1, Location::Any());
3899      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3900      break;
3901    }
3902    case Primitive::kPrimFloat:
3903    case Primitive::kPrimDouble: {
3904      locations->SetInAt(0, Location::RequiresFpuRegister());
3905      locations->SetInAt(1, Location::RequiresFpuRegister());
3906      locations->SetOut(Location::RequiresRegister());
3907      break;
3908    }
3909    default:
3910      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3911  }
3912}
3913
3914void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
3915  LocationSummary* locations = compare->GetLocations();
3916  Register out = locations->Out().AsRegister<Register>();
3917  Location left = locations->InAt(0);
3918  Location right = locations->InAt(1);
3919
3920  NearLabel less, greater, done;
3921  switch (compare->InputAt(0)->GetType()) {
3922    case Primitive::kPrimLong: {
3923      Register left_low = left.AsRegisterPairLow<Register>();
3924      Register left_high = left.AsRegisterPairHigh<Register>();
3925      int32_t val_low = 0;
3926      int32_t val_high = 0;
3927      bool right_is_const = false;
3928
3929      if (right.IsConstant()) {
3930        DCHECK(right.GetConstant()->IsLongConstant());
3931        right_is_const = true;
3932        int64_t val = right.GetConstant()->AsLongConstant()->GetValue();
3933        val_low = Low32Bits(val);
3934        val_high = High32Bits(val);
3935      }
3936
3937      if (right.IsRegisterPair()) {
3938        __ cmpl(left_high, right.AsRegisterPairHigh<Register>());
3939      } else if (right.IsDoubleStackSlot()) {
3940        __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
3941      } else {
3942        DCHECK(right_is_const) << right;
3943        if (val_high == 0) {
3944          __ testl(left_high, left_high);
3945        } else {
3946          __ cmpl(left_high, Immediate(val_high));
3947        }
3948      }
3949      __ j(kLess, &less);  // Signed compare.
3950      __ j(kGreater, &greater);  // Signed compare.
3951      if (right.IsRegisterPair()) {
3952        __ cmpl(left_low, right.AsRegisterPairLow<Register>());
3953      } else if (right.IsDoubleStackSlot()) {
3954        __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
3955      } else {
3956        DCHECK(right_is_const) << right;
3957        if (val_low == 0) {
3958          __ testl(left_low, left_low);
3959        } else {
3960          __ cmpl(left_low, Immediate(val_low));
3961        }
3962      }
3963      break;
3964    }
3965    case Primitive::kPrimFloat: {
3966      __ ucomiss(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3967      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3968      break;
3969    }
3970    case Primitive::kPrimDouble: {
3971      __ ucomisd(left.AsFpuRegister<XmmRegister>(), right.AsFpuRegister<XmmRegister>());
3972      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
3973      break;
3974    }
3975    default:
3976      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3977  }
3978  __ movl(out, Immediate(0));
3979  __ j(kEqual, &done);
3980  __ j(kBelow, &less);  // kBelow is for CF (unsigned & floats).
3981
3982  __ Bind(&greater);
3983  __ movl(out, Immediate(1));
3984  __ jmp(&done);
3985
3986  __ Bind(&less);
3987  __ movl(out, Immediate(-1));
3988
3989  __ Bind(&done);
3990}
3991
3992void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
3993  LocationSummary* locations =
3994      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3995  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3996    locations->SetInAt(i, Location::Any());
3997  }
3998  locations->SetOut(Location::Any());
3999}
4000
4001void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) {
4002  LOG(FATAL) << "Unreachable";
4003}
4004
4005void InstructionCodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
4006  /*
4007   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
4008   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
4009   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
4010   */
4011  switch (kind) {
4012    case MemBarrierKind::kAnyAny: {
4013      __ mfence();
4014      break;
4015    }
4016    case MemBarrierKind::kAnyStore:
4017    case MemBarrierKind::kLoadAny:
4018    case MemBarrierKind::kStoreStore: {
4019      // nop
4020      break;
4021    }
4022    default:
4023      LOG(FATAL) << "Unexpected memory barrier " << kind;
4024  }
4025}
4026
4027HInvokeStaticOrDirect::DispatchInfo CodeGeneratorX86::GetSupportedInvokeStaticOrDirectDispatch(
4028      const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
4029      MethodReference target_method ATTRIBUTE_UNUSED) {
4030  switch (desired_dispatch_info.code_ptr_location) {
4031    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
4032    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
4033      // For direct code, we actually prefer to call via the code pointer from ArtMethod*.
4034      // (Though the direct CALL ptr16:32 is available for consideration).
4035      return HInvokeStaticOrDirect::DispatchInfo {
4036        desired_dispatch_info.method_load_kind,
4037        HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod,
4038        desired_dispatch_info.method_load_data,
4039        0u
4040      };
4041    default:
4042      return desired_dispatch_info;
4043  }
4044}
4045
4046Register CodeGeneratorX86::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke,
4047                                                                 Register temp) {
4048  DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u);
4049  Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4050  if (!invoke->GetLocations()->Intrinsified()) {
4051    return location.AsRegister<Register>();
4052  }
4053  // For intrinsics we allow any location, so it may be on the stack.
4054  if (!location.IsRegister()) {
4055    __ movl(temp, Address(ESP, location.GetStackIndex()));
4056    return temp;
4057  }
4058  // For register locations, check if the register was saved. If so, get it from the stack.
4059  // Note: There is a chance that the register was saved but not overwritten, so we could
4060  // save one load. However, since this is just an intrinsic slow path we prefer this
4061  // simple and more robust approach rather that trying to determine if that's the case.
4062  SlowPathCode* slow_path = GetCurrentSlowPath();
4063  DCHECK(slow_path != nullptr);  // For intrinsified invokes the call is emitted on the slow path.
4064  if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) {
4065    int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>());
4066    __ movl(temp, Address(ESP, stack_offset));
4067    return temp;
4068  }
4069  return location.AsRegister<Register>();
4070}
4071
4072void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) {
4073  Location callee_method = temp;  // For all kinds except kRecursive, callee will be in temp.
4074  switch (invoke->GetMethodLoadKind()) {
4075    case HInvokeStaticOrDirect::MethodLoadKind::kStringInit:
4076      // temp = thread->string_init_entrypoint
4077      __ fs()->movl(temp.AsRegister<Register>(), Address::Absolute(invoke->GetStringInitOffset()));
4078      break;
4079    case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
4080      callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4081      break;
4082    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
4083      __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress()));
4084      break;
4085    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
4086      __ movl(temp.AsRegister<Register>(), Immediate(0));  // Placeholder.
4087      method_patches_.emplace_back(invoke->GetTargetMethod());
4088      __ Bind(&method_patches_.back().label);  // Bind the label at the end of the "movl" insn.
4089      break;
4090    case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
4091      Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke,
4092                                                                temp.AsRegister<Register>());
4093      uint32_t offset = invoke->GetDexCacheArrayOffset();
4094      __ movl(temp.AsRegister<Register>(), Address(base_reg, kDummy32BitOffset));
4095      // Add the patch entry and bind its label at the end of the instruction.
4096      pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, offset);
4097      __ Bind(&pc_relative_dex_cache_patches_.back().label);
4098      break;
4099    }
4100    case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
4101      Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4102      Register method_reg;
4103      Register reg = temp.AsRegister<Register>();
4104      if (current_method.IsRegister()) {
4105        method_reg = current_method.AsRegister<Register>();
4106      } else {
4107        DCHECK(IsBaseline() || invoke->GetLocations()->Intrinsified());
4108        DCHECK(!current_method.IsValid());
4109        method_reg = reg;
4110        __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
4111      }
4112      // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_;
4113      __ movl(reg, Address(method_reg,
4114                           ArtMethod::DexCacheResolvedMethodsOffset(kX86PointerSize).Int32Value()));
4115      // temp = temp[index_in_cache]
4116      uint32_t index_in_cache = invoke->GetTargetMethod().dex_method_index;
4117      __ movl(reg, Address(reg, CodeGenerator::GetCachePointerOffset(index_in_cache)));
4118      break;
4119    }
4120  }
4121
4122  switch (invoke->GetCodePtrLocation()) {
4123    case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf:
4124      __ call(GetFrameEntryLabel());
4125      break;
4126    case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: {
4127      relative_call_patches_.emplace_back(invoke->GetTargetMethod());
4128      Label* label = &relative_call_patches_.back().label;
4129      __ call(label);  // Bind to the patch label, override at link time.
4130      __ Bind(label);  // Bind the label at the end of the "call" insn.
4131      break;
4132    }
4133    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
4134    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
4135      // Filtered out by GetSupportedInvokeStaticOrDirectDispatch().
4136      LOG(FATAL) << "Unsupported";
4137      UNREACHABLE();
4138    case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
4139      // (callee_method + offset_of_quick_compiled_code)()
4140      __ call(Address(callee_method.AsRegister<Register>(),
4141                      ArtMethod::EntryPointFromQuickCompiledCodeOffset(
4142                          kX86WordSize).Int32Value()));
4143      break;
4144  }
4145
4146  DCHECK(!IsLeafMethod());
4147}
4148
4149void CodeGeneratorX86::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) {
4150  Register temp = temp_in.AsRegister<Register>();
4151  uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
4152      invoke->GetVTableIndex(), kX86PointerSize).Uint32Value();
4153  LocationSummary* locations = invoke->GetLocations();
4154  Location receiver = locations->InAt(0);
4155  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4156  DCHECK(receiver.IsRegister());
4157  // /* HeapReference<Class> */ temp = receiver->klass_
4158  __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
4159  MaybeRecordImplicitNullCheck(invoke);
4160  // Instead of simply (possibly) unpoisoning `temp` here, we should
4161  // emit a read barrier for the previous class reference load.
4162  // However this is not required in practice, as this is an
4163  // intermediate/temporary reference and because the current
4164  // concurrent copying collector keeps the from-space memory
4165  // intact/accessible until the end of the marking phase (the
4166  // concurrent copying collector may not in the future).
4167  __ MaybeUnpoisonHeapReference(temp);
4168  // temp = temp->GetMethodAt(method_offset);
4169  __ movl(temp, Address(temp, method_offset));
4170  // call temp->GetEntryPoint();
4171  __ call(Address(
4172      temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
4173}
4174
4175void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
4176  DCHECK(linker_patches->empty());
4177  size_t size =
4178      method_patches_.size() +
4179      relative_call_patches_.size() +
4180      pc_relative_dex_cache_patches_.size();
4181  linker_patches->reserve(size);
4182  // The label points to the end of the "movl" insn but the literal offset for method
4183  // patch needs to point to the embedded constant which occupies the last 4 bytes.
4184  constexpr uint32_t kLabelPositionToLiteralOffsetAdjustment = 4u;
4185  for (const MethodPatchInfo<Label>& info : method_patches_) {
4186    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4187    linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset,
4188                                                       info.target_method.dex_file,
4189                                                       info.target_method.dex_method_index));
4190  }
4191  for (const MethodPatchInfo<Label>& info : relative_call_patches_) {
4192    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4193    linker_patches->push_back(LinkerPatch::RelativeCodePatch(literal_offset,
4194                                                             info.target_method.dex_file,
4195                                                             info.target_method.dex_method_index));
4196  }
4197  for (const PcRelativeDexCacheAccessInfo& info : pc_relative_dex_cache_patches_) {
4198    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4199    linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(literal_offset,
4200                                                              &info.target_dex_file,
4201                                                              GetMethodAddressOffset(),
4202                                                              info.element_offset));
4203  }
4204}
4205
4206void CodeGeneratorX86::MarkGCCard(Register temp,
4207                                  Register card,
4208                                  Register object,
4209                                  Register value,
4210                                  bool value_can_be_null) {
4211  NearLabel is_null;
4212  if (value_can_be_null) {
4213    __ testl(value, value);
4214    __ j(kEqual, &is_null);
4215  }
4216  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
4217  __ movl(temp, object);
4218  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
4219  __ movb(Address(temp, card, TIMES_1, 0),
4220          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
4221  if (value_can_be_null) {
4222    __ Bind(&is_null);
4223  }
4224}
4225
4226void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
4227  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
4228
4229  bool object_field_get_with_read_barrier =
4230      kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
4231  LocationSummary* locations =
4232      new (GetGraph()->GetArena()) LocationSummary(instruction,
4233                                                   kEmitCompilerReadBarrier ?
4234                                                       LocationSummary::kCallOnSlowPath :
4235                                                       LocationSummary::kNoCall);
4236  locations->SetInAt(0, Location::RequiresRegister());
4237
4238  if (Primitive::IsFloatingPointType(instruction->GetType())) {
4239    locations->SetOut(Location::RequiresFpuRegister());
4240  } else {
4241    // The output overlaps in case of long: we don't want the low move
4242    // to overwrite the object's location.  Likewise, in the case of
4243    // an object field get with read barriers enabled, we do not want
4244    // the move to overwrite the object's location, as we need it to emit
4245    // the read barrier.
4246    locations->SetOut(
4247        Location::RequiresRegister(),
4248        (object_field_get_with_read_barrier || instruction->GetType() == Primitive::kPrimLong) ?
4249            Location::kOutputOverlap :
4250            Location::kNoOutputOverlap);
4251  }
4252
4253  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
4254    // Long values can be loaded atomically into an XMM using movsd.
4255    // So we use an XMM register as a temp to achieve atomicity (first load the temp into the XMM
4256    // and then copy the XMM into the output 32bits at a time).
4257    locations->AddTemp(Location::RequiresFpuRegister());
4258  }
4259}
4260
4261void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
4262                                                 const FieldInfo& field_info) {
4263  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
4264
4265  LocationSummary* locations = instruction->GetLocations();
4266  Location base_loc = locations->InAt(0);
4267  Register base = base_loc.AsRegister<Register>();
4268  Location out = locations->Out();
4269  bool is_volatile = field_info.IsVolatile();
4270  Primitive::Type field_type = field_info.GetFieldType();
4271  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
4272
4273  switch (field_type) {
4274    case Primitive::kPrimBoolean: {
4275      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
4276      break;
4277    }
4278
4279    case Primitive::kPrimByte: {
4280      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
4281      break;
4282    }
4283
4284    case Primitive::kPrimShort: {
4285      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
4286      break;
4287    }
4288
4289    case Primitive::kPrimChar: {
4290      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
4291      break;
4292    }
4293
4294    case Primitive::kPrimInt:
4295    case Primitive::kPrimNot: {
4296      __ movl(out.AsRegister<Register>(), Address(base, offset));
4297      break;
4298    }
4299
4300    case Primitive::kPrimLong: {
4301      if (is_volatile) {
4302        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
4303        __ movsd(temp, Address(base, offset));
4304        codegen_->MaybeRecordImplicitNullCheck(instruction);
4305        __ movd(out.AsRegisterPairLow<Register>(), temp);
4306        __ psrlq(temp, Immediate(32));
4307        __ movd(out.AsRegisterPairHigh<Register>(), temp);
4308      } else {
4309        DCHECK_NE(base, out.AsRegisterPairLow<Register>());
4310        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
4311        codegen_->MaybeRecordImplicitNullCheck(instruction);
4312        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
4313      }
4314      break;
4315    }
4316
4317    case Primitive::kPrimFloat: {
4318      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
4319      break;
4320    }
4321
4322    case Primitive::kPrimDouble: {
4323      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
4324      break;
4325    }
4326
4327    case Primitive::kPrimVoid:
4328      LOG(FATAL) << "Unreachable type " << field_type;
4329      UNREACHABLE();
4330  }
4331
4332  // Longs are handled in the switch.
4333  if (field_type != Primitive::kPrimLong) {
4334    codegen_->MaybeRecordImplicitNullCheck(instruction);
4335  }
4336
4337  if (is_volatile) {
4338    GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4339  }
4340
4341  if (field_type == Primitive::kPrimNot) {
4342    codegen_->MaybeGenerateReadBarrier(instruction, out, out, base_loc, offset);
4343  }
4344}
4345
4346void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
4347  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
4348
4349  LocationSummary* locations =
4350      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4351  locations->SetInAt(0, Location::RequiresRegister());
4352  bool is_volatile = field_info.IsVolatile();
4353  Primitive::Type field_type = field_info.GetFieldType();
4354  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
4355    || (field_type == Primitive::kPrimByte);
4356
4357  // The register allocator does not support multiple
4358  // inputs that die at entry with one in a specific register.
4359  if (is_byte_type) {
4360    // Ensure the value is in a byte register.
4361    locations->SetInAt(1, Location::RegisterLocation(EAX));
4362  } else if (Primitive::IsFloatingPointType(field_type)) {
4363    if (is_volatile && field_type == Primitive::kPrimDouble) {
4364      // In order to satisfy the semantics of volatile, this must be a single instruction store.
4365      locations->SetInAt(1, Location::RequiresFpuRegister());
4366    } else {
4367      locations->SetInAt(1, Location::FpuRegisterOrConstant(instruction->InputAt(1)));
4368    }
4369  } else if (is_volatile && field_type == Primitive::kPrimLong) {
4370    // In order to satisfy the semantics of volatile, this must be a single instruction store.
4371    locations->SetInAt(1, Location::RequiresRegister());
4372
4373    // 64bits value can be atomically written to an address with movsd and an XMM register.
4374    // We need two XMM registers because there's no easier way to (bit) copy a register pair
4375    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
4376    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
4377    // isolated cases when we need this it isn't worth adding the extra complexity.
4378    locations->AddTemp(Location::RequiresFpuRegister());
4379    locations->AddTemp(Location::RequiresFpuRegister());
4380  } else {
4381    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4382
4383    if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
4384      // Temporary registers for the write barrier.
4385      locations->AddTemp(Location::RequiresRegister());  // May be used for reference poisoning too.
4386      // Ensure the card is in a byte register.
4387      locations->AddTemp(Location::RegisterLocation(ECX));
4388    }
4389  }
4390}
4391
4392void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
4393                                                 const FieldInfo& field_info,
4394                                                 bool value_can_be_null) {
4395  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
4396
4397  LocationSummary* locations = instruction->GetLocations();
4398  Register base = locations->InAt(0).AsRegister<Register>();
4399  Location value = locations->InAt(1);
4400  bool is_volatile = field_info.IsVolatile();
4401  Primitive::Type field_type = field_info.GetFieldType();
4402  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
4403  bool needs_write_barrier =
4404      CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1));
4405
4406  if (is_volatile) {
4407    GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
4408  }
4409
4410  bool maybe_record_implicit_null_check_done = false;
4411
4412  switch (field_type) {
4413    case Primitive::kPrimBoolean:
4414    case Primitive::kPrimByte: {
4415      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
4416      break;
4417    }
4418
4419    case Primitive::kPrimShort:
4420    case Primitive::kPrimChar: {
4421      if (value.IsConstant()) {
4422        int16_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4423        __ movw(Address(base, offset), Immediate(v));
4424      } else {
4425        __ movw(Address(base, offset), value.AsRegister<Register>());
4426      }
4427      break;
4428    }
4429
4430    case Primitive::kPrimInt:
4431    case Primitive::kPrimNot: {
4432      if (kPoisonHeapReferences && needs_write_barrier) {
4433        // Note that in the case where `value` is a null reference,
4434        // we do not enter this block, as the reference does not
4435        // need poisoning.
4436        DCHECK_EQ(field_type, Primitive::kPrimNot);
4437        Register temp = locations->GetTemp(0).AsRegister<Register>();
4438        __ movl(temp, value.AsRegister<Register>());
4439        __ PoisonHeapReference(temp);
4440        __ movl(Address(base, offset), temp);
4441      } else if (value.IsConstant()) {
4442        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4443        __ movl(Address(base, offset), Immediate(v));
4444      } else {
4445        __ movl(Address(base, offset), value.AsRegister<Register>());
4446      }
4447      break;
4448    }
4449
4450    case Primitive::kPrimLong: {
4451      if (is_volatile) {
4452        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
4453        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
4454        __ movd(temp1, value.AsRegisterPairLow<Register>());
4455        __ movd(temp2, value.AsRegisterPairHigh<Register>());
4456        __ punpckldq(temp1, temp2);
4457        __ movsd(Address(base, offset), temp1);
4458        codegen_->MaybeRecordImplicitNullCheck(instruction);
4459      } else if (value.IsConstant()) {
4460        int64_t v = CodeGenerator::GetInt64ValueOf(value.GetConstant());
4461        __ movl(Address(base, offset), Immediate(Low32Bits(v)));
4462        codegen_->MaybeRecordImplicitNullCheck(instruction);
4463        __ movl(Address(base, kX86WordSize + offset), Immediate(High32Bits(v)));
4464      } else {
4465        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
4466        codegen_->MaybeRecordImplicitNullCheck(instruction);
4467        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
4468      }
4469      maybe_record_implicit_null_check_done = true;
4470      break;
4471    }
4472
4473    case Primitive::kPrimFloat: {
4474      if (value.IsConstant()) {
4475        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4476        __ movl(Address(base, offset), Immediate(v));
4477      } else {
4478        __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
4479      }
4480      break;
4481    }
4482
4483    case Primitive::kPrimDouble: {
4484      if (value.IsConstant()) {
4485        int64_t v = CodeGenerator::GetInt64ValueOf(value.GetConstant());
4486        __ movl(Address(base, offset), Immediate(Low32Bits(v)));
4487        codegen_->MaybeRecordImplicitNullCheck(instruction);
4488        __ movl(Address(base, kX86WordSize + offset), Immediate(High32Bits(v)));
4489        maybe_record_implicit_null_check_done = true;
4490      } else {
4491        __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
4492      }
4493      break;
4494    }
4495
4496    case Primitive::kPrimVoid:
4497      LOG(FATAL) << "Unreachable type " << field_type;
4498      UNREACHABLE();
4499  }
4500
4501  if (!maybe_record_implicit_null_check_done) {
4502    codegen_->MaybeRecordImplicitNullCheck(instruction);
4503  }
4504
4505  if (needs_write_barrier) {
4506    Register temp = locations->GetTemp(0).AsRegister<Register>();
4507    Register card = locations->GetTemp(1).AsRegister<Register>();
4508    codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>(), value_can_be_null);
4509  }
4510
4511  if (is_volatile) {
4512    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
4513  }
4514}
4515
4516void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4517  HandleFieldGet(instruction, instruction->GetFieldInfo());
4518}
4519
4520void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4521  HandleFieldGet(instruction, instruction->GetFieldInfo());
4522}
4523
4524void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4525  HandleFieldSet(instruction, instruction->GetFieldInfo());
4526}
4527
4528void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4529  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4530}
4531
4532void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4533  HandleFieldSet(instruction, instruction->GetFieldInfo());
4534}
4535
4536void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4537  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4538}
4539
4540void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4541  HandleFieldGet(instruction, instruction->GetFieldInfo());
4542}
4543
4544void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4545  HandleFieldGet(instruction, instruction->GetFieldInfo());
4546}
4547
4548void LocationsBuilderX86::VisitUnresolvedInstanceFieldGet(
4549    HUnresolvedInstanceFieldGet* instruction) {
4550  FieldAccessCallingConventionX86 calling_convention;
4551  codegen_->CreateUnresolvedFieldLocationSummary(
4552      instruction, instruction->GetFieldType(), calling_convention);
4553}
4554
4555void InstructionCodeGeneratorX86::VisitUnresolvedInstanceFieldGet(
4556    HUnresolvedInstanceFieldGet* instruction) {
4557  FieldAccessCallingConventionX86 calling_convention;
4558  codegen_->GenerateUnresolvedFieldAccess(instruction,
4559                                          instruction->GetFieldType(),
4560                                          instruction->GetFieldIndex(),
4561                                          instruction->GetDexPc(),
4562                                          calling_convention);
4563}
4564
4565void LocationsBuilderX86::VisitUnresolvedInstanceFieldSet(
4566    HUnresolvedInstanceFieldSet* instruction) {
4567  FieldAccessCallingConventionX86 calling_convention;
4568  codegen_->CreateUnresolvedFieldLocationSummary(
4569      instruction, instruction->GetFieldType(), calling_convention);
4570}
4571
4572void InstructionCodeGeneratorX86::VisitUnresolvedInstanceFieldSet(
4573    HUnresolvedInstanceFieldSet* instruction) {
4574  FieldAccessCallingConventionX86 calling_convention;
4575  codegen_->GenerateUnresolvedFieldAccess(instruction,
4576                                          instruction->GetFieldType(),
4577                                          instruction->GetFieldIndex(),
4578                                          instruction->GetDexPc(),
4579                                          calling_convention);
4580}
4581
4582void LocationsBuilderX86::VisitUnresolvedStaticFieldGet(
4583    HUnresolvedStaticFieldGet* instruction) {
4584  FieldAccessCallingConventionX86 calling_convention;
4585  codegen_->CreateUnresolvedFieldLocationSummary(
4586      instruction, instruction->GetFieldType(), calling_convention);
4587}
4588
4589void InstructionCodeGeneratorX86::VisitUnresolvedStaticFieldGet(
4590    HUnresolvedStaticFieldGet* instruction) {
4591  FieldAccessCallingConventionX86 calling_convention;
4592  codegen_->GenerateUnresolvedFieldAccess(instruction,
4593                                          instruction->GetFieldType(),
4594                                          instruction->GetFieldIndex(),
4595                                          instruction->GetDexPc(),
4596                                          calling_convention);
4597}
4598
4599void LocationsBuilderX86::VisitUnresolvedStaticFieldSet(
4600    HUnresolvedStaticFieldSet* instruction) {
4601  FieldAccessCallingConventionX86 calling_convention;
4602  codegen_->CreateUnresolvedFieldLocationSummary(
4603      instruction, instruction->GetFieldType(), calling_convention);
4604}
4605
4606void InstructionCodeGeneratorX86::VisitUnresolvedStaticFieldSet(
4607    HUnresolvedStaticFieldSet* instruction) {
4608  FieldAccessCallingConventionX86 calling_convention;
4609  codegen_->GenerateUnresolvedFieldAccess(instruction,
4610                                          instruction->GetFieldType(),
4611                                          instruction->GetFieldIndex(),
4612                                          instruction->GetDexPc(),
4613                                          calling_convention);
4614}
4615
4616void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
4617  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
4618      ? LocationSummary::kCallOnSlowPath
4619      : LocationSummary::kNoCall;
4620  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4621  Location loc = codegen_->IsImplicitNullCheckAllowed(instruction)
4622      ? Location::RequiresRegister()
4623      : Location::Any();
4624  locations->SetInAt(0, loc);
4625  if (instruction->HasUses()) {
4626    locations->SetOut(Location::SameAsFirstInput());
4627  }
4628}
4629
4630void InstructionCodeGeneratorX86::GenerateImplicitNullCheck(HNullCheck* instruction) {
4631  if (codegen_->CanMoveNullCheckToUser(instruction)) {
4632    return;
4633  }
4634  LocationSummary* locations = instruction->GetLocations();
4635  Location obj = locations->InAt(0);
4636
4637  __ testl(EAX, Address(obj.AsRegister<Register>(), 0));
4638  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4639}
4640
4641void InstructionCodeGeneratorX86::GenerateExplicitNullCheck(HNullCheck* instruction) {
4642  SlowPathCode* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
4643  codegen_->AddSlowPath(slow_path);
4644
4645  LocationSummary* locations = instruction->GetLocations();
4646  Location obj = locations->InAt(0);
4647
4648  if (obj.IsRegister()) {
4649    __ testl(obj.AsRegister<Register>(), obj.AsRegister<Register>());
4650  } else if (obj.IsStackSlot()) {
4651    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
4652  } else {
4653    DCHECK(obj.IsConstant()) << obj;
4654    DCHECK(obj.GetConstant()->IsNullConstant());
4655    __ jmp(slow_path->GetEntryLabel());
4656    return;
4657  }
4658  __ j(kEqual, slow_path->GetEntryLabel());
4659}
4660
4661void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
4662  if (codegen_->IsImplicitNullCheckAllowed(instruction)) {
4663    GenerateImplicitNullCheck(instruction);
4664  } else {
4665    GenerateExplicitNullCheck(instruction);
4666  }
4667}
4668
4669void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
4670  bool object_array_get_with_read_barrier =
4671      kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
4672  LocationSummary* locations =
4673      new (GetGraph()->GetArena()) LocationSummary(instruction,
4674                                                   object_array_get_with_read_barrier ?
4675                                                       LocationSummary::kCallOnSlowPath :
4676                                                       LocationSummary::kNoCall);
4677  locations->SetInAt(0, Location::RequiresRegister());
4678  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4679  if (Primitive::IsFloatingPointType(instruction->GetType())) {
4680    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
4681  } else {
4682    // The output overlaps in case of long: we don't want the low move
4683    // to overwrite the array's location.  Likewise, in the case of an
4684    // object array get with read barriers enabled, we do not want the
4685    // move to overwrite the array's location, as we need it to emit
4686    // the read barrier.
4687    locations->SetOut(
4688        Location::RequiresRegister(),
4689        (instruction->GetType() == Primitive::kPrimLong || object_array_get_with_read_barrier) ?
4690            Location::kOutputOverlap :
4691            Location::kNoOutputOverlap);
4692  }
4693}
4694
4695void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
4696  LocationSummary* locations = instruction->GetLocations();
4697  Location obj_loc = locations->InAt(0);
4698  Register obj = obj_loc.AsRegister<Register>();
4699  Location index = locations->InAt(1);
4700
4701  Primitive::Type type = instruction->GetType();
4702  switch (type) {
4703    case Primitive::kPrimBoolean: {
4704      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
4705      Register out = locations->Out().AsRegister<Register>();
4706      if (index.IsConstant()) {
4707        __ movzxb(out, Address(obj,
4708            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
4709      } else {
4710        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
4711      }
4712      break;
4713    }
4714
4715    case Primitive::kPrimByte: {
4716      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
4717      Register out = locations->Out().AsRegister<Register>();
4718      if (index.IsConstant()) {
4719        __ movsxb(out, Address(obj,
4720            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
4721      } else {
4722        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
4723      }
4724      break;
4725    }
4726
4727    case Primitive::kPrimShort: {
4728      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
4729      Register out = locations->Out().AsRegister<Register>();
4730      if (index.IsConstant()) {
4731        __ movsxw(out, Address(obj,
4732            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
4733      } else {
4734        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
4735      }
4736      break;
4737    }
4738
4739    case Primitive::kPrimChar: {
4740      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
4741      Register out = locations->Out().AsRegister<Register>();
4742      if (index.IsConstant()) {
4743        __ movzxw(out, Address(obj,
4744            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
4745      } else {
4746        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
4747      }
4748      break;
4749    }
4750
4751    case Primitive::kPrimInt:
4752    case Primitive::kPrimNot: {
4753      static_assert(
4754          sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
4755          "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
4756      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4757      Register out = locations->Out().AsRegister<Register>();
4758      if (index.IsConstant()) {
4759        __ movl(out, Address(obj,
4760            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
4761      } else {
4762        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
4763      }
4764      break;
4765    }
4766
4767    case Primitive::kPrimLong: {
4768      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
4769      Location out = locations->Out();
4770      DCHECK_NE(obj, out.AsRegisterPairLow<Register>());
4771      if (index.IsConstant()) {
4772        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4773        __ movl(out.AsRegisterPairLow<Register>(), Address(obj, offset));
4774        codegen_->MaybeRecordImplicitNullCheck(instruction);
4775        __ movl(out.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
4776      } else {
4777        __ movl(out.AsRegisterPairLow<Register>(),
4778                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
4779        codegen_->MaybeRecordImplicitNullCheck(instruction);
4780        __ movl(out.AsRegisterPairHigh<Register>(),
4781                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
4782      }
4783      break;
4784    }
4785
4786    case Primitive::kPrimFloat: {
4787      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
4788      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
4789      if (index.IsConstant()) {
4790        __ movss(out, Address(obj,
4791            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
4792      } else {
4793        __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
4794      }
4795      break;
4796    }
4797
4798    case Primitive::kPrimDouble: {
4799      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
4800      XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
4801      if (index.IsConstant()) {
4802        __ movsd(out, Address(obj,
4803            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
4804      } else {
4805        __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
4806      }
4807      break;
4808    }
4809
4810    case Primitive::kPrimVoid:
4811      LOG(FATAL) << "Unreachable type " << type;
4812      UNREACHABLE();
4813  }
4814
4815  if (type != Primitive::kPrimLong) {
4816    codegen_->MaybeRecordImplicitNullCheck(instruction);
4817  }
4818
4819  if (type == Primitive::kPrimNot) {
4820    static_assert(
4821        sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
4822        "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
4823    uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4824    Location out = locations->Out();
4825    if (index.IsConstant()) {
4826      uint32_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4827      codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, offset);
4828    } else {
4829      codegen_->MaybeGenerateReadBarrier(instruction, out, out, obj_loc, data_offset, index);
4830    }
4831  }
4832}
4833
4834void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
4835  // This location builder might end up asking to up to four registers, which is
4836  // not currently possible for baseline. The situation in which we need four
4837  // registers cannot be met by baseline though, because it has not run any
4838  // optimization.
4839
4840  Primitive::Type value_type = instruction->GetComponentType();
4841
4842  bool needs_write_barrier =
4843      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
4844  bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
4845  bool object_array_set_with_read_barrier =
4846      kEmitCompilerReadBarrier && (value_type == Primitive::kPrimNot);
4847
4848  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4849      instruction,
4850      (may_need_runtime_call_for_type_check || object_array_set_with_read_barrier) ?
4851          LocationSummary::kCallOnSlowPath :
4852          LocationSummary::kNoCall);
4853
4854  bool is_byte_type = (value_type == Primitive::kPrimBoolean)
4855      || (value_type == Primitive::kPrimByte);
4856  // We need the inputs to be different than the output in case of long operation.
4857  // In case of a byte operation, the register allocator does not support multiple
4858  // inputs that die at entry with one in a specific register.
4859  locations->SetInAt(0, Location::RequiresRegister());
4860  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4861  if (is_byte_type) {
4862    // Ensure the value is in a byte register.
4863    locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
4864  } else if (Primitive::IsFloatingPointType(value_type)) {
4865    locations->SetInAt(2, Location::FpuRegisterOrConstant(instruction->InputAt(2)));
4866  } else {
4867    locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
4868  }
4869  if (needs_write_barrier) {
4870    // Temporary registers for the write barrier.
4871    locations->AddTemp(Location::RequiresRegister());  // Possibly used for ref. poisoning too.
4872    // Ensure the card is in a byte register.
4873    locations->AddTemp(Location::RegisterLocation(ECX));
4874  }
4875}
4876
4877void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
4878  LocationSummary* locations = instruction->GetLocations();
4879  Location array_loc = locations->InAt(0);
4880  Register array = array_loc.AsRegister<Register>();
4881  Location index = locations->InAt(1);
4882  Location value = locations->InAt(2);
4883  Primitive::Type value_type = instruction->GetComponentType();
4884  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4885  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
4886  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
4887  bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
4888  bool needs_write_barrier =
4889      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
4890
4891  switch (value_type) {
4892    case Primitive::kPrimBoolean:
4893    case Primitive::kPrimByte: {
4894      uint32_t offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
4895      Address address = index.IsConstant()
4896          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + offset)
4897          : Address(array, index.AsRegister<Register>(), TIMES_1, offset);
4898      if (value.IsRegister()) {
4899        __ movb(address, value.AsRegister<ByteRegister>());
4900      } else {
4901        __ movb(address, Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4902      }
4903      codegen_->MaybeRecordImplicitNullCheck(instruction);
4904      break;
4905    }
4906
4907    case Primitive::kPrimShort:
4908    case Primitive::kPrimChar: {
4909      uint32_t offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
4910      Address address = index.IsConstant()
4911          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + offset)
4912          : Address(array, index.AsRegister<Register>(), TIMES_2, offset);
4913      if (value.IsRegister()) {
4914        __ movw(address, value.AsRegister<Register>());
4915      } else {
4916        __ movw(address, Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
4917      }
4918      codegen_->MaybeRecordImplicitNullCheck(instruction);
4919      break;
4920    }
4921
4922    case Primitive::kPrimNot: {
4923      uint32_t offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4924      Address address = index.IsConstant()
4925          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
4926          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
4927
4928      if (!value.IsRegister()) {
4929        // Just setting null.
4930        DCHECK(instruction->InputAt(2)->IsNullConstant());
4931        DCHECK(value.IsConstant()) << value;
4932        __ movl(address, Immediate(0));
4933        codegen_->MaybeRecordImplicitNullCheck(instruction);
4934        DCHECK(!needs_write_barrier);
4935        DCHECK(!may_need_runtime_call_for_type_check);
4936        break;
4937      }
4938
4939      DCHECK(needs_write_barrier);
4940      Register register_value = value.AsRegister<Register>();
4941      NearLabel done, not_null, do_put;
4942      SlowPathCode* slow_path = nullptr;
4943      Register temp = locations->GetTemp(0).AsRegister<Register>();
4944      if (may_need_runtime_call_for_type_check) {
4945        slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathX86(instruction);
4946        codegen_->AddSlowPath(slow_path);
4947        if (instruction->GetValueCanBeNull()) {
4948          __ testl(register_value, register_value);
4949          __ j(kNotEqual, &not_null);
4950          __ movl(address, Immediate(0));
4951          codegen_->MaybeRecordImplicitNullCheck(instruction);
4952          __ jmp(&done);
4953          __ Bind(&not_null);
4954        }
4955
4956        if (kEmitCompilerReadBarrier) {
4957          // When read barriers are enabled, the type checking
4958          // instrumentation requires two read barriers:
4959          //
4960          //   __ movl(temp2, temp);
4961          //   // /* HeapReference<Class> */ temp = temp->component_type_
4962          //   __ movl(temp, Address(temp, component_offset));
4963          //   codegen_->GenerateReadBarrier(
4964          //       instruction, temp_loc, temp_loc, temp2_loc, component_offset);
4965          //
4966          //   // /* HeapReference<Class> */ temp2 = register_value->klass_
4967          //   __ movl(temp2, Address(register_value, class_offset));
4968          //   codegen_->GenerateReadBarrier(
4969          //       instruction, temp2_loc, temp2_loc, value, class_offset, temp_loc);
4970          //
4971          //   __ cmpl(temp, temp2);
4972          //
4973          // However, the second read barrier may trash `temp`, as it
4974          // is a temporary register, and as such would not be saved
4975          // along with live registers before calling the runtime (nor
4976          // restored afterwards).  So in this case, we bail out and
4977          // delegate the work to the array set slow path.
4978          //
4979          // TODO: Extend the register allocator to support a new
4980          // "(locally) live temp" location so as to avoid always
4981          // going into the slow path when read barriers are enabled.
4982          __ jmp(slow_path->GetEntryLabel());
4983        } else {
4984          // /* HeapReference<Class> */ temp = array->klass_
4985          __ movl(temp, Address(array, class_offset));
4986          codegen_->MaybeRecordImplicitNullCheck(instruction);
4987          __ MaybeUnpoisonHeapReference(temp);
4988
4989          // /* HeapReference<Class> */ temp = temp->component_type_
4990          __ movl(temp, Address(temp, component_offset));
4991          // If heap poisoning is enabled, no need to unpoison `temp`
4992          // nor the object reference in `register_value->klass`, as
4993          // we are comparing two poisoned references.
4994          __ cmpl(temp, Address(register_value, class_offset));
4995
4996          if (instruction->StaticTypeOfArrayIsObjectArray()) {
4997            __ j(kEqual, &do_put);
4998            // If heap poisoning is enabled, the `temp` reference has
4999            // not been unpoisoned yet; unpoison it now.
5000            __ MaybeUnpoisonHeapReference(temp);
5001
5002            // /* HeapReference<Class> */ temp = temp->super_class_
5003            __ movl(temp, Address(temp, super_offset));
5004            // If heap poisoning is enabled, no need to unpoison
5005            // `temp`, as we are comparing against null below.
5006            __ testl(temp, temp);
5007            __ j(kNotEqual, slow_path->GetEntryLabel());
5008            __ Bind(&do_put);
5009          } else {
5010            __ j(kNotEqual, slow_path->GetEntryLabel());
5011          }
5012        }
5013      }
5014
5015      if (kPoisonHeapReferences) {
5016        __ movl(temp, register_value);
5017        __ PoisonHeapReference(temp);
5018        __ movl(address, temp);
5019      } else {
5020        __ movl(address, register_value);
5021      }
5022      if (!may_need_runtime_call_for_type_check) {
5023        codegen_->MaybeRecordImplicitNullCheck(instruction);
5024      }
5025
5026      Register card = locations->GetTemp(1).AsRegister<Register>();
5027      codegen_->MarkGCCard(
5028          temp, card, array, value.AsRegister<Register>(), instruction->GetValueCanBeNull());
5029      __ Bind(&done);
5030
5031      if (slow_path != nullptr) {
5032        __ Bind(slow_path->GetExitLabel());
5033      }
5034
5035      break;
5036    }
5037
5038    case Primitive::kPrimInt: {
5039      uint32_t offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
5040      Address address = index.IsConstant()
5041          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5042          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5043      if (value.IsRegister()) {
5044        __ movl(address, value.AsRegister<Register>());
5045      } else {
5046        DCHECK(value.IsConstant()) << value;
5047        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
5048        __ movl(address, Immediate(v));
5049      }
5050      codegen_->MaybeRecordImplicitNullCheck(instruction);
5051      break;
5052    }
5053
5054    case Primitive::kPrimLong: {
5055      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
5056      if (index.IsConstant()) {
5057        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
5058        if (value.IsRegisterPair()) {
5059          __ movl(Address(array, offset), value.AsRegisterPairLow<Register>());
5060          codegen_->MaybeRecordImplicitNullCheck(instruction);
5061          __ movl(Address(array, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
5062        } else {
5063          DCHECK(value.IsConstant());
5064          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
5065          __ movl(Address(array, offset), Immediate(Low32Bits(val)));
5066          codegen_->MaybeRecordImplicitNullCheck(instruction);
5067          __ movl(Address(array, offset + kX86WordSize), Immediate(High32Bits(val)));
5068        }
5069      } else {
5070        if (value.IsRegisterPair()) {
5071          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset),
5072                  value.AsRegisterPairLow<Register>());
5073          codegen_->MaybeRecordImplicitNullCheck(instruction);
5074          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
5075                  value.AsRegisterPairHigh<Register>());
5076        } else {
5077          DCHECK(value.IsConstant());
5078          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
5079          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset),
5080                  Immediate(Low32Bits(val)));
5081          codegen_->MaybeRecordImplicitNullCheck(instruction);
5082          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
5083                  Immediate(High32Bits(val)));
5084        }
5085      }
5086      break;
5087    }
5088
5089    case Primitive::kPrimFloat: {
5090      uint32_t offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
5091      Address address = index.IsConstant()
5092          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5093          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5094      if (value.IsFpuRegister()) {
5095        __ movss(address, value.AsFpuRegister<XmmRegister>());
5096      } else {
5097        DCHECK(value.IsConstant());
5098        int32_t v = bit_cast<int32_t, float>(value.GetConstant()->AsFloatConstant()->GetValue());
5099        __ movl(address, Immediate(v));
5100      }
5101      codegen_->MaybeRecordImplicitNullCheck(instruction);
5102      break;
5103    }
5104
5105    case Primitive::kPrimDouble: {
5106      uint32_t offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
5107      Address address = index.IsConstant()
5108          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + offset)
5109          : Address(array, index.AsRegister<Register>(), TIMES_8, offset);
5110      if (value.IsFpuRegister()) {
5111        __ movsd(address, value.AsFpuRegister<XmmRegister>());
5112      } else {
5113        DCHECK(value.IsConstant());
5114        Address address_hi = index.IsConstant() ?
5115            Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) +
5116                           offset + kX86WordSize) :
5117            Address(array, index.AsRegister<Register>(), TIMES_8, offset + kX86WordSize);
5118        int64_t v = bit_cast<int64_t, double>(value.GetConstant()->AsDoubleConstant()->GetValue());
5119        __ movl(address, Immediate(Low32Bits(v)));
5120        codegen_->MaybeRecordImplicitNullCheck(instruction);
5121        __ movl(address_hi, Immediate(High32Bits(v)));
5122      }
5123      break;
5124    }
5125
5126    case Primitive::kPrimVoid:
5127      LOG(FATAL) << "Unreachable type " << instruction->GetType();
5128      UNREACHABLE();
5129  }
5130}
5131
5132void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
5133  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
5134  locations->SetInAt(0, Location::RequiresRegister());
5135  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
5136}
5137
5138void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
5139  LocationSummary* locations = instruction->GetLocations();
5140  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
5141  Register obj = locations->InAt(0).AsRegister<Register>();
5142  Register out = locations->Out().AsRegister<Register>();
5143  __ movl(out, Address(obj, offset));
5144  codegen_->MaybeRecordImplicitNullCheck(instruction);
5145}
5146
5147void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
5148  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
5149      ? LocationSummary::kCallOnSlowPath
5150      : LocationSummary::kNoCall;
5151  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5152  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
5153  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
5154  if (instruction->HasUses()) {
5155    locations->SetOut(Location::SameAsFirstInput());
5156  }
5157}
5158
5159void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
5160  LocationSummary* locations = instruction->GetLocations();
5161  Location index_loc = locations->InAt(0);
5162  Location length_loc = locations->InAt(1);
5163  SlowPathCode* slow_path =
5164    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(instruction);
5165
5166  if (length_loc.IsConstant()) {
5167    int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
5168    if (index_loc.IsConstant()) {
5169      // BCE will remove the bounds check if we are guarenteed to pass.
5170      int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
5171      if (index < 0 || index >= length) {
5172        codegen_->AddSlowPath(slow_path);
5173        __ jmp(slow_path->GetEntryLabel());
5174      } else {
5175        // Some optimization after BCE may have generated this, and we should not
5176        // generate a bounds check if it is a valid range.
5177      }
5178      return;
5179    }
5180
5181    // We have to reverse the jump condition because the length is the constant.
5182    Register index_reg = index_loc.AsRegister<Register>();
5183    __ cmpl(index_reg, Immediate(length));
5184    codegen_->AddSlowPath(slow_path);
5185    __ j(kAboveEqual, slow_path->GetEntryLabel());
5186  } else {
5187    Register length = length_loc.AsRegister<Register>();
5188    if (index_loc.IsConstant()) {
5189      int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
5190      __ cmpl(length, Immediate(value));
5191    } else {
5192      __ cmpl(length, index_loc.AsRegister<Register>());
5193    }
5194    codegen_->AddSlowPath(slow_path);
5195    __ j(kBelowEqual, slow_path->GetEntryLabel());
5196  }
5197}
5198
5199void LocationsBuilderX86::VisitTemporary(HTemporary* temp) {
5200  temp->SetLocations(nullptr);
5201}
5202
5203void InstructionCodeGeneratorX86::VisitTemporary(HTemporary* temp ATTRIBUTE_UNUSED) {
5204  // Nothing to do, this is driven by the code generator.
5205}
5206
5207void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) {
5208  LOG(FATAL) << "Unreachable";
5209}
5210
5211void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
5212  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
5213}
5214
5215void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
5216  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
5217}
5218
5219void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
5220  HBasicBlock* block = instruction->GetBlock();
5221  if (block->GetLoopInformation() != nullptr) {
5222    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
5223    // The back edge will generate the suspend check.
5224    return;
5225  }
5226  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
5227    // The goto will generate the suspend check.
5228    return;
5229  }
5230  GenerateSuspendCheck(instruction, nullptr);
5231}
5232
5233void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
5234                                                       HBasicBlock* successor) {
5235  SuspendCheckSlowPathX86* slow_path =
5236      down_cast<SuspendCheckSlowPathX86*>(instruction->GetSlowPath());
5237  if (slow_path == nullptr) {
5238    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
5239    instruction->SetSlowPath(slow_path);
5240    codegen_->AddSlowPath(slow_path);
5241    if (successor != nullptr) {
5242      DCHECK(successor->IsLoopHeader());
5243      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
5244    }
5245  } else {
5246    DCHECK_EQ(slow_path->GetSuccessor(), successor);
5247  }
5248
5249  __ fs()->cmpw(Address::Absolute(
5250      Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()), Immediate(0));
5251  if (successor == nullptr) {
5252    __ j(kNotEqual, slow_path->GetEntryLabel());
5253    __ Bind(slow_path->GetReturnLabel());
5254  } else {
5255    __ j(kEqual, codegen_->GetLabelOf(successor));
5256    __ jmp(slow_path->GetEntryLabel());
5257  }
5258}
5259
5260X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
5261  return codegen_->GetAssembler();
5262}
5263
5264void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) {
5265  ScratchRegisterScope ensure_scratch(
5266      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5267  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5268  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5269  __ movl(temp_reg, Address(ESP, src + stack_offset));
5270  __ movl(Address(ESP, dst + stack_offset), temp_reg);
5271}
5272
5273void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) {
5274  ScratchRegisterScope ensure_scratch(
5275      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5276  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5277  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5278  __ movl(temp_reg, Address(ESP, src + stack_offset));
5279  __ movl(Address(ESP, dst + stack_offset), temp_reg);
5280  __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize));
5281  __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg);
5282}
5283
5284void ParallelMoveResolverX86::EmitMove(size_t index) {
5285  MoveOperands* move = moves_[index];
5286  Location source = move->GetSource();
5287  Location destination = move->GetDestination();
5288
5289  if (source.IsRegister()) {
5290    if (destination.IsRegister()) {
5291      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5292    } else {
5293      DCHECK(destination.IsStackSlot());
5294      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
5295    }
5296  } else if (source.IsFpuRegister()) {
5297    if (destination.IsFpuRegister()) {
5298      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5299    } else if (destination.IsStackSlot()) {
5300      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
5301    } else {
5302      DCHECK(destination.IsDoubleStackSlot());
5303      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
5304    }
5305  } else if (source.IsStackSlot()) {
5306    if (destination.IsRegister()) {
5307      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
5308    } else if (destination.IsFpuRegister()) {
5309      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
5310    } else {
5311      DCHECK(destination.IsStackSlot());
5312      MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex());
5313    }
5314  } else if (source.IsDoubleStackSlot()) {
5315    if (destination.IsFpuRegister()) {
5316      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
5317    } else {
5318      DCHECK(destination.IsDoubleStackSlot()) << destination;
5319      MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex());
5320    }
5321  } else if (source.IsConstant()) {
5322    HConstant* constant = source.GetConstant();
5323    if (constant->IsIntConstant() || constant->IsNullConstant()) {
5324      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
5325      if (destination.IsRegister()) {
5326        if (value == 0) {
5327          __ xorl(destination.AsRegister<Register>(), destination.AsRegister<Register>());
5328        } else {
5329          __ movl(destination.AsRegister<Register>(), Immediate(value));
5330        }
5331      } else {
5332        DCHECK(destination.IsStackSlot()) << destination;
5333        __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
5334      }
5335    } else if (constant->IsFloatConstant()) {
5336      float fp_value = constant->AsFloatConstant()->GetValue();
5337      int32_t value = bit_cast<int32_t, float>(fp_value);
5338      Immediate imm(value);
5339      if (destination.IsFpuRegister()) {
5340        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
5341        if (value == 0) {
5342          // Easy handling of 0.0.
5343          __ xorps(dest, dest);
5344        } else {
5345          ScratchRegisterScope ensure_scratch(
5346              this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5347          Register temp = static_cast<Register>(ensure_scratch.GetRegister());
5348          __ movl(temp, Immediate(value));
5349          __ movd(dest, temp);
5350        }
5351      } else {
5352        DCHECK(destination.IsStackSlot()) << destination;
5353        __ movl(Address(ESP, destination.GetStackIndex()), imm);
5354      }
5355    } else if (constant->IsLongConstant()) {
5356      int64_t value = constant->AsLongConstant()->GetValue();
5357      int32_t low_value = Low32Bits(value);
5358      int32_t high_value = High32Bits(value);
5359      Immediate low(low_value);
5360      Immediate high(high_value);
5361      if (destination.IsDoubleStackSlot()) {
5362        __ movl(Address(ESP, destination.GetStackIndex()), low);
5363        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
5364      } else {
5365        __ movl(destination.AsRegisterPairLow<Register>(), low);
5366        __ movl(destination.AsRegisterPairHigh<Register>(), high);
5367      }
5368    } else {
5369      DCHECK(constant->IsDoubleConstant());
5370      double dbl_value = constant->AsDoubleConstant()->GetValue();
5371      int64_t value = bit_cast<int64_t, double>(dbl_value);
5372      int32_t low_value = Low32Bits(value);
5373      int32_t high_value = High32Bits(value);
5374      Immediate low(low_value);
5375      Immediate high(high_value);
5376      if (destination.IsFpuRegister()) {
5377        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
5378        if (value == 0) {
5379          // Easy handling of 0.0.
5380          __ xorpd(dest, dest);
5381        } else {
5382          __ pushl(high);
5383          __ pushl(low);
5384          __ movsd(dest, Address(ESP, 0));
5385          __ addl(ESP, Immediate(8));
5386        }
5387      } else {
5388        DCHECK(destination.IsDoubleStackSlot()) << destination;
5389        __ movl(Address(ESP, destination.GetStackIndex()), low);
5390        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
5391      }
5392    }
5393  } else {
5394    LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
5395  }
5396}
5397
5398void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
5399  Register suggested_scratch = reg == EAX ? EBX : EAX;
5400  ScratchRegisterScope ensure_scratch(
5401      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
5402
5403  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5404  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
5405  __ movl(Address(ESP, mem + stack_offset), reg);
5406  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
5407}
5408
5409void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) {
5410  ScratchRegisterScope ensure_scratch(
5411      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5412
5413  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5414  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5415  __ movl(temp_reg, Address(ESP, mem + stack_offset));
5416  __ movss(Address(ESP, mem + stack_offset), reg);
5417  __ movd(reg, temp_reg);
5418}
5419
5420void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
5421  ScratchRegisterScope ensure_scratch1(
5422      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5423
5424  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
5425  ScratchRegisterScope ensure_scratch2(
5426      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
5427
5428  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
5429  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
5430  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
5431  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
5432  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
5433  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
5434}
5435
5436void ParallelMoveResolverX86::EmitSwap(size_t index) {
5437  MoveOperands* move = moves_[index];
5438  Location source = move->GetSource();
5439  Location destination = move->GetDestination();
5440
5441  if (source.IsRegister() && destination.IsRegister()) {
5442    // Use XOR swap algorithm to avoid serializing XCHG instruction or using a temporary.
5443    DCHECK_NE(destination.AsRegister<Register>(), source.AsRegister<Register>());
5444    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5445    __ xorl(source.AsRegister<Register>(), destination.AsRegister<Register>());
5446    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5447  } else if (source.IsRegister() && destination.IsStackSlot()) {
5448    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
5449  } else if (source.IsStackSlot() && destination.IsRegister()) {
5450    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
5451  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
5452    Exchange(destination.GetStackIndex(), source.GetStackIndex());
5453  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
5454    // Use XOR Swap algorithm to avoid a temporary.
5455    DCHECK_NE(source.reg(), destination.reg());
5456    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5457    __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
5458    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5459  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
5460    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
5461  } else if (destination.IsFpuRegister() && source.IsStackSlot()) {
5462    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
5463  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
5464    // Take advantage of the 16 bytes in the XMM register.
5465    XmmRegister reg = source.AsFpuRegister<XmmRegister>();
5466    Address stack(ESP, destination.GetStackIndex());
5467    // Load the double into the high doubleword.
5468    __ movhpd(reg, stack);
5469
5470    // Store the low double into the destination.
5471    __ movsd(stack, reg);
5472
5473    // Move the high double to the low double.
5474    __ psrldq(reg, Immediate(8));
5475  } else if (destination.IsFpuRegister() && source.IsDoubleStackSlot()) {
5476    // Take advantage of the 16 bytes in the XMM register.
5477    XmmRegister reg = destination.AsFpuRegister<XmmRegister>();
5478    Address stack(ESP, source.GetStackIndex());
5479    // Load the double into the high doubleword.
5480    __ movhpd(reg, stack);
5481
5482    // Store the low double into the destination.
5483    __ movsd(stack, reg);
5484
5485    // Move the high double to the low double.
5486    __ psrldq(reg, Immediate(8));
5487  } else if (destination.IsDoubleStackSlot() && source.IsDoubleStackSlot()) {
5488    Exchange(destination.GetStackIndex(), source.GetStackIndex());
5489    Exchange(destination.GetHighStackIndex(kX86WordSize), source.GetHighStackIndex(kX86WordSize));
5490  } else {
5491    LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination;
5492  }
5493}
5494
5495void ParallelMoveResolverX86::SpillScratch(int reg) {
5496  __ pushl(static_cast<Register>(reg));
5497}
5498
5499void ParallelMoveResolverX86::RestoreScratch(int reg) {
5500  __ popl(static_cast<Register>(reg));
5501}
5502
5503void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
5504  InvokeRuntimeCallingConvention calling_convention;
5505  CodeGenerator::CreateLoadClassLocationSummary(
5506      cls,
5507      Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
5508      Location::RegisterLocation(EAX),
5509      /* code_generator_supports_read_barrier */ true);
5510}
5511
5512void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
5513  LocationSummary* locations = cls->GetLocations();
5514  if (cls->NeedsAccessCheck()) {
5515    codegen_->MoveConstant(locations->GetTemp(0), cls->GetTypeIndex());
5516    codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInitializeTypeAndVerifyAccess),
5517                            cls,
5518                            cls->GetDexPc(),
5519                            nullptr);
5520    CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>();
5521    return;
5522  }
5523
5524  Location out_loc = locations->Out();
5525  Register out = out_loc.AsRegister<Register>();
5526  Register current_method = locations->InAt(0).AsRegister<Register>();
5527
5528  if (cls->IsReferrersClass()) {
5529    DCHECK(!cls->CanCallRuntime());
5530    DCHECK(!cls->MustGenerateClinitCheck());
5531    uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value();
5532    if (kEmitCompilerReadBarrier) {
5533      // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_)
5534      __ leal(out, Address(current_method, declaring_class_offset));
5535      // /* mirror::Class* */ out = out->Read()
5536      codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc);
5537    } else {
5538      // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
5539      __ movl(out, Address(current_method, declaring_class_offset));
5540    }
5541  } else {
5542    // /* GcRoot<mirror::Class>[] */ out =
5543    //        current_method.ptr_sized_fields_->dex_cache_resolved_types_
5544    __ movl(out, Address(current_method,
5545                         ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value()));
5546
5547    size_t cache_offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex());
5548    if (kEmitCompilerReadBarrier) {
5549      // /* GcRoot<mirror::Class>* */ out = &out[type_index]
5550      __ leal(out, Address(out, cache_offset));
5551      // /* mirror::Class* */ out = out->Read()
5552      codegen_->GenerateReadBarrierForRoot(cls, out_loc, out_loc);
5553    } else {
5554      // /* GcRoot<mirror::Class> */ out = out[type_index]
5555      __ movl(out, Address(out, cache_offset));
5556    }
5557
5558    if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) {
5559      DCHECK(cls->CanCallRuntime());
5560      SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
5561          cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
5562      codegen_->AddSlowPath(slow_path);
5563
5564      if (!cls->IsInDexCache()) {
5565        __ testl(out, out);
5566        __ j(kEqual, slow_path->GetEntryLabel());
5567      }
5568
5569      if (cls->MustGenerateClinitCheck()) {
5570        GenerateClassInitializationCheck(slow_path, out);
5571      } else {
5572        __ Bind(slow_path->GetExitLabel());
5573      }
5574    }
5575  }
5576}
5577
5578void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
5579  LocationSummary* locations =
5580      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
5581  locations->SetInAt(0, Location::RequiresRegister());
5582  if (check->HasUses()) {
5583    locations->SetOut(Location::SameAsFirstInput());
5584  }
5585}
5586
5587void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
5588  // We assume the class to not be null.
5589  SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
5590      check->GetLoadClass(), check, check->GetDexPc(), true);
5591  codegen_->AddSlowPath(slow_path);
5592  GenerateClassInitializationCheck(slow_path,
5593                                   check->GetLocations()->InAt(0).AsRegister<Register>());
5594}
5595
5596void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
5597    SlowPathCode* slow_path, Register class_reg) {
5598  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
5599          Immediate(mirror::Class::kStatusInitialized));
5600  __ j(kLess, slow_path->GetEntryLabel());
5601  __ Bind(slow_path->GetExitLabel());
5602  // No need for memory fence, thanks to the X86 memory model.
5603}
5604
5605void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
5606  LocationSummary* locations =
5607      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
5608  locations->SetInAt(0, Location::RequiresRegister());
5609  locations->SetOut(Location::RequiresRegister());
5610}
5611
5612void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
5613  SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
5614  codegen_->AddSlowPath(slow_path);
5615
5616  LocationSummary* locations = load->GetLocations();
5617  Location out_loc = locations->Out();
5618  Register out = out_loc.AsRegister<Register>();
5619  Register current_method = locations->InAt(0).AsRegister<Register>();
5620
5621  uint32_t declaring_class_offset = ArtMethod::DeclaringClassOffset().Int32Value();
5622  if (kEmitCompilerReadBarrier) {
5623    // /* GcRoot<mirror::Class>* */ out = &(current_method->declaring_class_)
5624    __ leal(out, Address(current_method, declaring_class_offset));
5625    // /* mirror::Class* */ out = out->Read()
5626    codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc);
5627  } else {
5628    // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
5629    __ movl(out, Address(current_method, declaring_class_offset));
5630  }
5631
5632  // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
5633  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
5634
5635  size_t cache_offset = CodeGenerator::GetCacheOffset(load->GetStringIndex());
5636  if (kEmitCompilerReadBarrier) {
5637    // /* GcRoot<mirror::String>* */ out = &out[string_index]
5638    __ leal(out, Address(out, cache_offset));
5639    // /* mirror::String* */ out = out->Read()
5640    codegen_->GenerateReadBarrierForRoot(load, out_loc, out_loc);
5641  } else {
5642    // /* GcRoot<mirror::String> */ out = out[string_index]
5643    __ movl(out, Address(out, cache_offset));
5644  }
5645
5646  __ testl(out, out);
5647  __ j(kEqual, slow_path->GetEntryLabel());
5648  __ Bind(slow_path->GetExitLabel());
5649}
5650
5651static Address GetExceptionTlsAddress() {
5652  return Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
5653}
5654
5655void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
5656  LocationSummary* locations =
5657      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
5658  locations->SetOut(Location::RequiresRegister());
5659}
5660
5661void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
5662  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), GetExceptionTlsAddress());
5663}
5664
5665void LocationsBuilderX86::VisitClearException(HClearException* clear) {
5666  new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
5667}
5668
5669void InstructionCodeGeneratorX86::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
5670  __ fs()->movl(GetExceptionTlsAddress(), Immediate(0));
5671}
5672
5673void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
5674  LocationSummary* locations =
5675      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
5676  InvokeRuntimeCallingConvention calling_convention;
5677  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
5678}
5679
5680void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
5681  codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pDeliverException),
5682                          instruction,
5683                          instruction->GetDexPc(),
5684                          nullptr);
5685  CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>();
5686}
5687
5688void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
5689  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
5690  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5691  switch (type_check_kind) {
5692    case TypeCheckKind::kExactCheck:
5693    case TypeCheckKind::kAbstractClassCheck:
5694    case TypeCheckKind::kClassHierarchyCheck:
5695    case TypeCheckKind::kArrayObjectCheck:
5696      call_kind =
5697          kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall;
5698      break;
5699    case TypeCheckKind::kArrayCheck:
5700    case TypeCheckKind::kUnresolvedCheck:
5701    case TypeCheckKind::kInterfaceCheck:
5702      call_kind = LocationSummary::kCallOnSlowPath;
5703      break;
5704  }
5705
5706  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5707  locations->SetInAt(0, Location::RequiresRegister());
5708  locations->SetInAt(1, Location::Any());
5709  // Note that TypeCheckSlowPathX86 uses this "out" register too.
5710  locations->SetOut(Location::RequiresRegister());
5711  // When read barriers are enabled, we need a temporary register for
5712  // some cases.
5713  if (kEmitCompilerReadBarrier &&
5714      (type_check_kind == TypeCheckKind::kAbstractClassCheck ||
5715       type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
5716       type_check_kind == TypeCheckKind::kArrayObjectCheck)) {
5717    locations->AddTemp(Location::RequiresRegister());
5718  }
5719}
5720
5721void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
5722  LocationSummary* locations = instruction->GetLocations();
5723  Location obj_loc = locations->InAt(0);
5724  Register obj = obj_loc.AsRegister<Register>();
5725  Location cls = locations->InAt(1);
5726  Location out_loc = locations->Out();
5727  Register out = out_loc.AsRegister<Register>();
5728  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5729  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
5730  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
5731  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
5732  SlowPathCode* slow_path = nullptr;
5733  NearLabel done, zero;
5734
5735  // Return 0 if `obj` is null.
5736  // Avoid null check if we know obj is not null.
5737  if (instruction->MustDoNullCheck()) {
5738    __ testl(obj, obj);
5739    __ j(kEqual, &zero);
5740  }
5741
5742  // /* HeapReference<Class> */ out = obj->klass_
5743  __ movl(out, Address(obj, class_offset));
5744  codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, obj_loc, class_offset);
5745
5746  switch (instruction->GetTypeCheckKind()) {
5747    case TypeCheckKind::kExactCheck: {
5748      if (cls.IsRegister()) {
5749        __ cmpl(out, cls.AsRegister<Register>());
5750      } else {
5751        DCHECK(cls.IsStackSlot()) << cls;
5752        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5753      }
5754
5755      // Classes must be equal for the instanceof to succeed.
5756      __ j(kNotEqual, &zero);
5757      __ movl(out, Immediate(1));
5758      __ jmp(&done);
5759      break;
5760    }
5761
5762    case TypeCheckKind::kAbstractClassCheck: {
5763      // If the class is abstract, we eagerly fetch the super class of the
5764      // object to avoid doing a comparison we know will fail.
5765      NearLabel loop;
5766      __ Bind(&loop);
5767      Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
5768      if (kEmitCompilerReadBarrier) {
5769        // Save the value of `out` into `temp` before overwriting it
5770        // in the following move operation, as we will need it for the
5771        // read barrier below.
5772        Register temp = temp_loc.AsRegister<Register>();
5773        __ movl(temp, out);
5774      }
5775      // /* HeapReference<Class> */ out = out->super_class_
5776      __ movl(out, Address(out, super_offset));
5777      codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset);
5778      __ testl(out, out);
5779      // If `out` is null, we use it for the result, and jump to `done`.
5780      __ j(kEqual, &done);
5781      if (cls.IsRegister()) {
5782        __ cmpl(out, cls.AsRegister<Register>());
5783      } else {
5784        DCHECK(cls.IsStackSlot()) << cls;
5785        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5786      }
5787      __ j(kNotEqual, &loop);
5788      __ movl(out, Immediate(1));
5789      if (zero.IsLinked()) {
5790        __ jmp(&done);
5791      }
5792      break;
5793    }
5794
5795    case TypeCheckKind::kClassHierarchyCheck: {
5796      // Walk over the class hierarchy to find a match.
5797      NearLabel loop, success;
5798      __ Bind(&loop);
5799      if (cls.IsRegister()) {
5800        __ cmpl(out, cls.AsRegister<Register>());
5801      } else {
5802        DCHECK(cls.IsStackSlot()) << cls;
5803        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5804      }
5805      __ j(kEqual, &success);
5806      Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
5807      if (kEmitCompilerReadBarrier) {
5808        // Save the value of `out` into `temp` before overwriting it
5809        // in the following move operation, as we will need it for the
5810        // read barrier below.
5811        Register temp = temp_loc.AsRegister<Register>();
5812        __ movl(temp, out);
5813      }
5814      // /* HeapReference<Class> */ out = out->super_class_
5815      __ movl(out, Address(out, super_offset));
5816      codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, super_offset);
5817      __ testl(out, out);
5818      __ j(kNotEqual, &loop);
5819      // If `out` is null, we use it for the result, and jump to `done`.
5820      __ jmp(&done);
5821      __ Bind(&success);
5822      __ movl(out, Immediate(1));
5823      if (zero.IsLinked()) {
5824        __ jmp(&done);
5825      }
5826      break;
5827    }
5828
5829    case TypeCheckKind::kArrayObjectCheck: {
5830      // Do an exact check.
5831      NearLabel exact_check;
5832      if (cls.IsRegister()) {
5833        __ cmpl(out, cls.AsRegister<Register>());
5834      } else {
5835        DCHECK(cls.IsStackSlot()) << cls;
5836        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5837      }
5838      __ j(kEqual, &exact_check);
5839      // Otherwise, we need to check that the object's class is a non-primitive array.
5840      Location temp_loc = kEmitCompilerReadBarrier ? locations->GetTemp(0) : Location::NoLocation();
5841      if (kEmitCompilerReadBarrier) {
5842        // Save the value of `out` into `temp` before overwriting it
5843        // in the following move operation, as we will need it for the
5844        // read barrier below.
5845        Register temp = temp_loc.AsRegister<Register>();
5846        __ movl(temp, out);
5847      }
5848      // /* HeapReference<Class> */ out = out->component_type_
5849      __ movl(out, Address(out, component_offset));
5850      codegen_->MaybeGenerateReadBarrier(instruction, out_loc, out_loc, temp_loc, component_offset);
5851      __ testl(out, out);
5852      // If `out` is null, we use it for the result, and jump to `done`.
5853      __ j(kEqual, &done);
5854      __ cmpw(Address(out, primitive_offset), Immediate(Primitive::kPrimNot));
5855      __ j(kNotEqual, &zero);
5856      __ Bind(&exact_check);
5857      __ movl(out, Immediate(1));
5858      __ jmp(&done);
5859      break;
5860    }
5861
5862    case TypeCheckKind::kArrayCheck: {
5863      if (cls.IsRegister()) {
5864        __ cmpl(out, cls.AsRegister<Register>());
5865      } else {
5866        DCHECK(cls.IsStackSlot()) << cls;
5867        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
5868      }
5869      DCHECK(locations->OnlyCallsOnSlowPath());
5870      slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
5871                                                                    /* is_fatal */ false);
5872      codegen_->AddSlowPath(slow_path);
5873      __ j(kNotEqual, slow_path->GetEntryLabel());
5874      __ movl(out, Immediate(1));
5875      if (zero.IsLinked()) {
5876        __ jmp(&done);
5877      }
5878      break;
5879    }
5880
5881    case TypeCheckKind::kUnresolvedCheck:
5882    case TypeCheckKind::kInterfaceCheck: {
5883      // Note that we indeed only call on slow path, but we always go
5884      // into the slow path for the unresolved & interface check
5885      // cases.
5886      //
5887      // We cannot directly call the InstanceofNonTrivial runtime
5888      // entry point without resorting to a type checking slow path
5889      // here (i.e. by calling InvokeRuntime directly), as it would
5890      // require to assign fixed registers for the inputs of this
5891      // HInstanceOf instruction (following the runtime calling
5892      // convention), which might be cluttered by the potential first
5893      // read barrier emission at the beginning of this method.
5894      DCHECK(locations->OnlyCallsOnSlowPath());
5895      slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
5896                                                                    /* is_fatal */ false);
5897      codegen_->AddSlowPath(slow_path);
5898      __ jmp(slow_path->GetEntryLabel());
5899      if (zero.IsLinked()) {
5900        __ jmp(&done);
5901      }
5902      break;
5903    }
5904  }
5905
5906  if (zero.IsLinked()) {
5907    __ Bind(&zero);
5908    __ xorl(out, out);
5909  }
5910
5911  if (done.IsLinked()) {
5912    __ Bind(&done);
5913  }
5914
5915  if (slow_path != nullptr) {
5916    __ Bind(slow_path->GetExitLabel());
5917  }
5918}
5919
5920void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
5921  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
5922  bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
5923  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5924  switch (type_check_kind) {
5925    case TypeCheckKind::kExactCheck:
5926    case TypeCheckKind::kAbstractClassCheck:
5927    case TypeCheckKind::kClassHierarchyCheck:
5928    case TypeCheckKind::kArrayObjectCheck:
5929      call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ?
5930          LocationSummary::kCallOnSlowPath :
5931          LocationSummary::kNoCall;  // In fact, call on a fatal (non-returning) slow path.
5932      break;
5933    case TypeCheckKind::kArrayCheck:
5934    case TypeCheckKind::kUnresolvedCheck:
5935    case TypeCheckKind::kInterfaceCheck:
5936      call_kind = LocationSummary::kCallOnSlowPath;
5937      break;
5938  }
5939  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5940  locations->SetInAt(0, Location::RequiresRegister());
5941  locations->SetInAt(1, Location::Any());
5942  // Note that TypeCheckSlowPathX86 uses this "temp" register too.
5943  locations->AddTemp(Location::RequiresRegister());
5944  // When read barriers are enabled, we need an additional temporary
5945  // register for some cases.
5946  if (kEmitCompilerReadBarrier &&
5947      (type_check_kind == TypeCheckKind::kAbstractClassCheck ||
5948       type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
5949       type_check_kind == TypeCheckKind::kArrayObjectCheck)) {
5950    locations->AddTemp(Location::RequiresRegister());
5951  }
5952}
5953
5954void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
5955  LocationSummary* locations = instruction->GetLocations();
5956  Location obj_loc = locations->InAt(0);
5957  Register obj = obj_loc.AsRegister<Register>();
5958  Location cls = locations->InAt(1);
5959  Location temp_loc = locations->GetTemp(0);
5960  Register temp = temp_loc.AsRegister<Register>();
5961  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5962  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
5963  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
5964  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
5965
5966  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5967  bool is_type_check_slow_path_fatal =
5968      (type_check_kind == TypeCheckKind::kExactCheck ||
5969       type_check_kind == TypeCheckKind::kAbstractClassCheck ||
5970       type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
5971       type_check_kind == TypeCheckKind::kArrayObjectCheck) &&
5972      !instruction->CanThrowIntoCatchBlock();
5973  SlowPathCode* type_check_slow_path =
5974      new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
5975                                                        is_type_check_slow_path_fatal);
5976  codegen_->AddSlowPath(type_check_slow_path);
5977
5978  NearLabel done;
5979  // Avoid null check if we know obj is not null.
5980  if (instruction->MustDoNullCheck()) {
5981    __ testl(obj, obj);
5982    __ j(kEqual, &done);
5983  }
5984
5985  // /* HeapReference<Class> */ temp = obj->klass_
5986  __ movl(temp, Address(obj, class_offset));
5987  codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
5988
5989  switch (type_check_kind) {
5990    case TypeCheckKind::kExactCheck:
5991    case TypeCheckKind::kArrayCheck: {
5992      if (cls.IsRegister()) {
5993        __ cmpl(temp, cls.AsRegister<Register>());
5994      } else {
5995        DCHECK(cls.IsStackSlot()) << cls;
5996        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
5997      }
5998      // Jump to slow path for throwing the exception or doing a
5999      // more involved array check.
6000      __ j(kNotEqual, type_check_slow_path->GetEntryLabel());
6001      break;
6002    }
6003
6004    case TypeCheckKind::kAbstractClassCheck: {
6005      // If the class is abstract, we eagerly fetch the super class of the
6006      // object to avoid doing a comparison we know will fail.
6007      NearLabel loop, compare_classes;
6008      __ Bind(&loop);
6009      Location temp2_loc =
6010          kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
6011      if (kEmitCompilerReadBarrier) {
6012        // Save the value of `temp` into `temp2` before overwriting it
6013        // in the following move operation, as we will need it for the
6014        // read barrier below.
6015        Register temp2 = temp2_loc.AsRegister<Register>();
6016        __ movl(temp2, temp);
6017      }
6018      // /* HeapReference<Class> */ temp = temp->super_class_
6019      __ movl(temp, Address(temp, super_offset));
6020      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset);
6021
6022      // If the class reference currently in `temp` is not null, jump
6023      // to the `compare_classes` label to compare it with the checked
6024      // class.
6025      __ testl(temp, temp);
6026      __ j(kNotEqual, &compare_classes);
6027      // Otherwise, jump to the slow path to throw the exception.
6028      //
6029      // But before, move back the object's class into `temp` before
6030      // going into the slow path, as it has been overwritten in the
6031      // meantime.
6032      // /* HeapReference<Class> */ temp = obj->klass_
6033      __ movl(temp, Address(obj, class_offset));
6034      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
6035      __ jmp(type_check_slow_path->GetEntryLabel());
6036
6037      __ Bind(&compare_classes);
6038      if (cls.IsRegister()) {
6039        __ cmpl(temp, cls.AsRegister<Register>());
6040      } else {
6041        DCHECK(cls.IsStackSlot()) << cls;
6042        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6043      }
6044      __ j(kNotEqual, &loop);
6045      break;
6046    }
6047
6048    case TypeCheckKind::kClassHierarchyCheck: {
6049      // Walk over the class hierarchy to find a match.
6050      NearLabel loop;
6051      __ Bind(&loop);
6052      if (cls.IsRegister()) {
6053        __ cmpl(temp, cls.AsRegister<Register>());
6054      } else {
6055        DCHECK(cls.IsStackSlot()) << cls;
6056        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6057      }
6058      __ j(kEqual, &done);
6059
6060      Location temp2_loc =
6061          kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
6062      if (kEmitCompilerReadBarrier) {
6063        // Save the value of `temp` into `temp2` before overwriting it
6064        // in the following move operation, as we will need it for the
6065        // read barrier below.
6066        Register temp2 = temp2_loc.AsRegister<Register>();
6067        __ movl(temp2, temp);
6068      }
6069      // /* HeapReference<Class> */ temp = temp->super_class_
6070      __ movl(temp, Address(temp, super_offset));
6071      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, temp2_loc, super_offset);
6072
6073      // If the class reference currently in `temp` is not null, jump
6074      // back at the beginning of the loop.
6075      __ testl(temp, temp);
6076      __ j(kNotEqual, &loop);
6077      // Otherwise, jump to the slow path to throw the exception.
6078      //
6079      // But before, move back the object's class into `temp` before
6080      // going into the slow path, as it has been overwritten in the
6081      // meantime.
6082      // /* HeapReference<Class> */ temp = obj->klass_
6083      __ movl(temp, Address(obj, class_offset));
6084      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
6085      __ jmp(type_check_slow_path->GetEntryLabel());
6086      break;
6087    }
6088
6089    case TypeCheckKind::kArrayObjectCheck: {
6090      // Do an exact check.
6091      NearLabel check_non_primitive_component_type;
6092      if (cls.IsRegister()) {
6093        __ cmpl(temp, cls.AsRegister<Register>());
6094      } else {
6095        DCHECK(cls.IsStackSlot()) << cls;
6096        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6097      }
6098      __ j(kEqual, &done);
6099
6100      // Otherwise, we need to check that the object's class is a non-primitive array.
6101      Location temp2_loc =
6102          kEmitCompilerReadBarrier ? locations->GetTemp(1) : Location::NoLocation();
6103      if (kEmitCompilerReadBarrier) {
6104        // Save the value of `temp` into `temp2` before overwriting it
6105        // in the following move operation, as we will need it for the
6106        // read barrier below.
6107        Register temp2 = temp2_loc.AsRegister<Register>();
6108        __ movl(temp2, temp);
6109      }
6110      // /* HeapReference<Class> */ temp = temp->component_type_
6111      __ movl(temp, Address(temp, component_offset));
6112      codegen_->MaybeGenerateReadBarrier(
6113          instruction, temp_loc, temp_loc, temp2_loc, component_offset);
6114
6115      // If the component type is not null (i.e. the object is indeed
6116      // an array), jump to label `check_non_primitive_component_type`
6117      // to further check that this component type is not a primitive
6118      // type.
6119      __ testl(temp, temp);
6120      __ j(kNotEqual, &check_non_primitive_component_type);
6121      // Otherwise, jump to the slow path to throw the exception.
6122      //
6123      // But before, move back the object's class into `temp` before
6124      // going into the slow path, as it has been overwritten in the
6125      // meantime.
6126      // /* HeapReference<Class> */ temp = obj->klass_
6127      __ movl(temp, Address(obj, class_offset));
6128      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
6129      __ jmp(type_check_slow_path->GetEntryLabel());
6130
6131      __ Bind(&check_non_primitive_component_type);
6132      __ cmpw(Address(temp, primitive_offset), Immediate(Primitive::kPrimNot));
6133      __ j(kEqual, &done);
6134      // Same comment as above regarding `temp` and the slow path.
6135      // /* HeapReference<Class> */ temp = obj->klass_
6136      __ movl(temp, Address(obj, class_offset));
6137      codegen_->MaybeGenerateReadBarrier(instruction, temp_loc, temp_loc, obj_loc, class_offset);
6138      __ jmp(type_check_slow_path->GetEntryLabel());
6139      break;
6140    }
6141
6142    case TypeCheckKind::kUnresolvedCheck:
6143    case TypeCheckKind::kInterfaceCheck:
6144      // We always go into the type check slow path for the unresolved &
6145      // interface check cases.
6146      //
6147      // We cannot directly call the CheckCast runtime entry point
6148      // without resorting to a type checking slow path here (i.e. by
6149      // calling InvokeRuntime directly), as it would require to
6150      // assign fixed registers for the inputs of this HInstanceOf
6151      // instruction (following the runtime calling convention), which
6152      // might be cluttered by the potential first read barrier
6153      // emission at the beginning of this method.
6154      __ jmp(type_check_slow_path->GetEntryLabel());
6155      break;
6156  }
6157  __ Bind(&done);
6158
6159  __ Bind(type_check_slow_path->GetExitLabel());
6160}
6161
6162void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
6163  LocationSummary* locations =
6164      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
6165  InvokeRuntimeCallingConvention calling_convention;
6166  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
6167}
6168
6169void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
6170  codegen_->InvokeRuntime(instruction->IsEnter() ? QUICK_ENTRY_POINT(pLockObject)
6171                                                 : QUICK_ENTRY_POINT(pUnlockObject),
6172                          instruction,
6173                          instruction->GetDexPc(),
6174                          nullptr);
6175  if (instruction->IsEnter()) {
6176    CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>();
6177  } else {
6178    CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>();
6179  }
6180}
6181
6182void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
6183void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
6184void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
6185
6186void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
6187  LocationSummary* locations =
6188      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
6189  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
6190         || instruction->GetResultType() == Primitive::kPrimLong);
6191  locations->SetInAt(0, Location::RequiresRegister());
6192  locations->SetInAt(1, Location::Any());
6193  locations->SetOut(Location::SameAsFirstInput());
6194}
6195
6196void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
6197  HandleBitwiseOperation(instruction);
6198}
6199
6200void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
6201  HandleBitwiseOperation(instruction);
6202}
6203
6204void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
6205  HandleBitwiseOperation(instruction);
6206}
6207
6208void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
6209  LocationSummary* locations = instruction->GetLocations();
6210  Location first = locations->InAt(0);
6211  Location second = locations->InAt(1);
6212  DCHECK(first.Equals(locations->Out()));
6213
6214  if (instruction->GetResultType() == Primitive::kPrimInt) {
6215    if (second.IsRegister()) {
6216      if (instruction->IsAnd()) {
6217        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
6218      } else if (instruction->IsOr()) {
6219        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
6220      } else {
6221        DCHECK(instruction->IsXor());
6222        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
6223      }
6224    } else if (second.IsConstant()) {
6225      if (instruction->IsAnd()) {
6226        __ andl(first.AsRegister<Register>(),
6227                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6228      } else if (instruction->IsOr()) {
6229        __ orl(first.AsRegister<Register>(),
6230               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6231      } else {
6232        DCHECK(instruction->IsXor());
6233        __ xorl(first.AsRegister<Register>(),
6234                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6235      }
6236    } else {
6237      if (instruction->IsAnd()) {
6238        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6239      } else if (instruction->IsOr()) {
6240        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6241      } else {
6242        DCHECK(instruction->IsXor());
6243        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6244      }
6245    }
6246  } else {
6247    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
6248    if (second.IsRegisterPair()) {
6249      if (instruction->IsAnd()) {
6250        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6251        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6252      } else if (instruction->IsOr()) {
6253        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6254        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6255      } else {
6256        DCHECK(instruction->IsXor());
6257        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6258        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6259      }
6260    } else if (second.IsDoubleStackSlot()) {
6261      if (instruction->IsAnd()) {
6262        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6263        __ andl(first.AsRegisterPairHigh<Register>(),
6264                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6265      } else if (instruction->IsOr()) {
6266        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6267        __ orl(first.AsRegisterPairHigh<Register>(),
6268                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6269      } else {
6270        DCHECK(instruction->IsXor());
6271        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6272        __ xorl(first.AsRegisterPairHigh<Register>(),
6273                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6274      }
6275    } else {
6276      DCHECK(second.IsConstant()) << second;
6277      int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
6278      int32_t low_value = Low32Bits(value);
6279      int32_t high_value = High32Bits(value);
6280      Immediate low(low_value);
6281      Immediate high(high_value);
6282      Register first_low = first.AsRegisterPairLow<Register>();
6283      Register first_high = first.AsRegisterPairHigh<Register>();
6284      if (instruction->IsAnd()) {
6285        if (low_value == 0) {
6286          __ xorl(first_low, first_low);
6287        } else if (low_value != -1) {
6288          __ andl(first_low, low);
6289        }
6290        if (high_value == 0) {
6291          __ xorl(first_high, first_high);
6292        } else if (high_value != -1) {
6293          __ andl(first_high, high);
6294        }
6295      } else if (instruction->IsOr()) {
6296        if (low_value != 0) {
6297          __ orl(first_low, low);
6298        }
6299        if (high_value != 0) {
6300          __ orl(first_high, high);
6301        }
6302      } else {
6303        DCHECK(instruction->IsXor());
6304        if (low_value != 0) {
6305          __ xorl(first_low, low);
6306        }
6307        if (high_value != 0) {
6308          __ xorl(first_high, high);
6309        }
6310      }
6311    }
6312  }
6313}
6314
6315void CodeGeneratorX86::GenerateReadBarrier(HInstruction* instruction,
6316                                           Location out,
6317                                           Location ref,
6318                                           Location obj,
6319                                           uint32_t offset,
6320                                           Location index) {
6321  DCHECK(kEmitCompilerReadBarrier);
6322
6323  // If heap poisoning is enabled, the unpoisoning of the loaded
6324  // reference will be carried out by the runtime within the slow
6325  // path.
6326  //
6327  // Note that `ref` currently does not get unpoisoned (when heap
6328  // poisoning is enabled), which is alright as the `ref` argument is
6329  // not used by the artReadBarrierSlow entry point.
6330  //
6331  // TODO: Unpoison `ref` when it is used by artReadBarrierSlow.
6332  SlowPathCode* slow_path = new (GetGraph()->GetArena())
6333      ReadBarrierForHeapReferenceSlowPathX86(instruction, out, ref, obj, offset, index);
6334  AddSlowPath(slow_path);
6335
6336  // TODO: When read barrier has a fast path, add it here.
6337  /* Currently the read barrier call is inserted after the original load.
6338   * However, if we have a fast path, we need to perform the load of obj.LockWord *before* the
6339   * original load. This load-load ordering is required by the read barrier.
6340   * The fast path/slow path (for Baker's algorithm) should look like:
6341   *
6342   * bool isGray = obj.LockWord & kReadBarrierMask;
6343   * lfence;  // load fence or artificial data dependence to prevent load-load reordering
6344   * ref = obj.field;    // this is the original load
6345   * if (isGray) {
6346   *   ref = Mark(ref);  // ideally the slow path just does Mark(ref)
6347   * }
6348   */
6349
6350  __ jmp(slow_path->GetEntryLabel());
6351  __ Bind(slow_path->GetExitLabel());
6352}
6353
6354void CodeGeneratorX86::MaybeGenerateReadBarrier(HInstruction* instruction,
6355                                                Location out,
6356                                                Location ref,
6357                                                Location obj,
6358                                                uint32_t offset,
6359                                                Location index) {
6360  if (kEmitCompilerReadBarrier) {
6361    // If heap poisoning is enabled, unpoisoning will be taken care of
6362    // by the runtime within the slow path.
6363    GenerateReadBarrier(instruction, out, ref, obj, offset, index);
6364  } else if (kPoisonHeapReferences) {
6365    __ UnpoisonHeapReference(out.AsRegister<Register>());
6366  }
6367}
6368
6369void CodeGeneratorX86::GenerateReadBarrierForRoot(HInstruction* instruction,
6370                                                  Location out,
6371                                                  Location root) {
6372  DCHECK(kEmitCompilerReadBarrier);
6373
6374  // Note that GC roots are not affected by heap poisoning, so we do
6375  // not need to do anything special for this here.
6376  SlowPathCode* slow_path =
6377      new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathX86(instruction, out, root);
6378  AddSlowPath(slow_path);
6379
6380  // TODO: Implement a fast path for ReadBarrierForRoot, performing
6381  // the following operation (for Baker's algorithm):
6382  //
6383  //   if (thread.tls32_.is_gc_marking) {
6384  //     root = Mark(root);
6385  //   }
6386
6387  __ jmp(slow_path->GetEntryLabel());
6388  __ Bind(slow_path->GetExitLabel());
6389}
6390
6391void LocationsBuilderX86::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
6392  // Nothing to do, this should be removed during prepare for register allocator.
6393  LOG(FATAL) << "Unreachable";
6394}
6395
6396void InstructionCodeGeneratorX86::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
6397  // Nothing to do, this should be removed during prepare for register allocator.
6398  LOG(FATAL) << "Unreachable";
6399}
6400
6401void LocationsBuilderX86::VisitFakeString(HFakeString* instruction) {
6402  DCHECK(codegen_->IsBaseline());
6403  LocationSummary* locations =
6404      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
6405  locations->SetOut(Location::ConstantLocation(GetGraph()->GetNullConstant()));
6406}
6407
6408void InstructionCodeGeneratorX86::VisitFakeString(HFakeString* instruction ATTRIBUTE_UNUSED) {
6409  DCHECK(codegen_->IsBaseline());
6410  // Will be generated at use site.
6411}
6412
6413// Simple implementation of packed switch - generate cascaded compare/jumps.
6414void LocationsBuilderX86::VisitPackedSwitch(HPackedSwitch* switch_instr) {
6415  LocationSummary* locations =
6416      new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
6417  locations->SetInAt(0, Location::RequiresRegister());
6418}
6419
6420void InstructionCodeGeneratorX86::VisitPackedSwitch(HPackedSwitch* switch_instr) {
6421  int32_t lower_bound = switch_instr->GetStartValue();
6422  int32_t num_entries = switch_instr->GetNumEntries();
6423  LocationSummary* locations = switch_instr->GetLocations();
6424  Register value_reg = locations->InAt(0).AsRegister<Register>();
6425  HBasicBlock* default_block = switch_instr->GetDefaultBlock();
6426
6427  // Create a series of compare/jumps.
6428  const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors();
6429  for (int i = 0; i < num_entries; i++) {
6430    int32_t case_value = lower_bound + i;
6431    if (case_value == 0) {
6432      __ testl(value_reg, value_reg);
6433    } else {
6434      __ cmpl(value_reg, Immediate(case_value));
6435    }
6436    __ j(kEqual, codegen_->GetLabelOf(successors[i]));
6437  }
6438
6439  // And the default for any other value.
6440  if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) {
6441      __ jmp(codegen_->GetLabelOf(default_block));
6442  }
6443}
6444
6445void LocationsBuilderX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) {
6446  LocationSummary* locations =
6447      new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
6448  locations->SetInAt(0, Location::RequiresRegister());
6449
6450  // Constant area pointer.
6451  locations->SetInAt(1, Location::RequiresRegister());
6452
6453  // And the temporary we need.
6454  locations->AddTemp(Location::RequiresRegister());
6455}
6456
6457void InstructionCodeGeneratorX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) {
6458  int32_t lower_bound = switch_instr->GetStartValue();
6459  int32_t num_entries = switch_instr->GetNumEntries();
6460  LocationSummary* locations = switch_instr->GetLocations();
6461  Register value_reg = locations->InAt(0).AsRegister<Register>();
6462  HBasicBlock* default_block = switch_instr->GetDefaultBlock();
6463
6464  // Optimizing has a jump area.
6465  Register temp_reg = locations->GetTemp(0).AsRegister<Register>();
6466  Register constant_area = locations->InAt(1).AsRegister<Register>();
6467
6468  // Remove the bias, if needed.
6469  if (lower_bound != 0) {
6470    __ leal(temp_reg, Address(value_reg, -lower_bound));
6471    value_reg = temp_reg;
6472  }
6473
6474  // Is the value in range?
6475  DCHECK_GE(num_entries, 1);
6476  __ cmpl(value_reg, Immediate(num_entries - 1));
6477  __ j(kAbove, codegen_->GetLabelOf(default_block));
6478
6479  // We are in the range of the table.
6480  // Load (target-constant_area) from the jump table, indexing by the value.
6481  __ movl(temp_reg, codegen_->LiteralCaseTable(switch_instr, constant_area, value_reg));
6482
6483  // Compute the actual target address by adding in constant_area.
6484  __ addl(temp_reg, constant_area);
6485
6486  // And jump.
6487  __ jmp(temp_reg);
6488}
6489
6490void LocationsBuilderX86::VisitX86ComputeBaseMethodAddress(
6491    HX86ComputeBaseMethodAddress* insn) {
6492  LocationSummary* locations =
6493      new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
6494  locations->SetOut(Location::RequiresRegister());
6495}
6496
6497void InstructionCodeGeneratorX86::VisitX86ComputeBaseMethodAddress(
6498    HX86ComputeBaseMethodAddress* insn) {
6499  LocationSummary* locations = insn->GetLocations();
6500  Register reg = locations->Out().AsRegister<Register>();
6501
6502  // Generate call to next instruction.
6503  Label next_instruction;
6504  __ call(&next_instruction);
6505  __ Bind(&next_instruction);
6506
6507  // Remember this offset for later use with constant area.
6508  codegen_->SetMethodAddressOffset(GetAssembler()->CodeSize());
6509
6510  // Grab the return address off the stack.
6511  __ popl(reg);
6512}
6513
6514void LocationsBuilderX86::VisitX86LoadFromConstantTable(
6515    HX86LoadFromConstantTable* insn) {
6516  LocationSummary* locations =
6517      new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
6518
6519  locations->SetInAt(0, Location::RequiresRegister());
6520  locations->SetInAt(1, Location::ConstantLocation(insn->GetConstant()));
6521
6522  // If we don't need to be materialized, we only need the inputs to be set.
6523  if (!insn->NeedsMaterialization()) {
6524    return;
6525  }
6526
6527  switch (insn->GetType()) {
6528    case Primitive::kPrimFloat:
6529    case Primitive::kPrimDouble:
6530      locations->SetOut(Location::RequiresFpuRegister());
6531      break;
6532
6533    case Primitive::kPrimInt:
6534      locations->SetOut(Location::RequiresRegister());
6535      break;
6536
6537    default:
6538      LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
6539  }
6540}
6541
6542void InstructionCodeGeneratorX86::VisitX86LoadFromConstantTable(HX86LoadFromConstantTable* insn) {
6543  if (!insn->NeedsMaterialization()) {
6544    return;
6545  }
6546
6547  LocationSummary* locations = insn->GetLocations();
6548  Location out = locations->Out();
6549  Register const_area = locations->InAt(0).AsRegister<Register>();
6550  HConstant *value = insn->GetConstant();
6551
6552  switch (insn->GetType()) {
6553    case Primitive::kPrimFloat:
6554      __ movss(out.AsFpuRegister<XmmRegister>(),
6555               codegen_->LiteralFloatAddress(value->AsFloatConstant()->GetValue(), const_area));
6556      break;
6557
6558    case Primitive::kPrimDouble:
6559      __ movsd(out.AsFpuRegister<XmmRegister>(),
6560               codegen_->LiteralDoubleAddress(value->AsDoubleConstant()->GetValue(), const_area));
6561      break;
6562
6563    case Primitive::kPrimInt:
6564      __ movl(out.AsRegister<Register>(),
6565              codegen_->LiteralInt32Address(value->AsIntConstant()->GetValue(), const_area));
6566      break;
6567
6568    default:
6569      LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
6570  }
6571}
6572
6573/**
6574 * Class to handle late fixup of offsets into constant area.
6575 */
6576class RIPFixup : public AssemblerFixup, public ArenaObject<kArenaAllocCodeGenerator> {
6577 public:
6578  RIPFixup(CodeGeneratorX86& codegen, size_t offset)
6579      : codegen_(&codegen), offset_into_constant_area_(offset) {}
6580
6581 protected:
6582  void SetOffset(size_t offset) { offset_into_constant_area_ = offset; }
6583
6584  CodeGeneratorX86* codegen_;
6585
6586 private:
6587  void Process(const MemoryRegion& region, int pos) OVERRIDE {
6588    // Patch the correct offset for the instruction.  The place to patch is the
6589    // last 4 bytes of the instruction.
6590    // The value to patch is the distance from the offset in the constant area
6591    // from the address computed by the HX86ComputeBaseMethodAddress instruction.
6592    int32_t constant_offset = codegen_->ConstantAreaStart() + offset_into_constant_area_;
6593    int32_t relative_position = constant_offset - codegen_->GetMethodAddressOffset();;
6594
6595    // Patch in the right value.
6596    region.StoreUnaligned<int32_t>(pos - 4, relative_position);
6597  }
6598
6599  // Location in constant area that the fixup refers to.
6600  int32_t offset_into_constant_area_;
6601};
6602
6603/**
6604 * Class to handle late fixup of offsets to a jump table that will be created in the
6605 * constant area.
6606 */
6607class JumpTableRIPFixup : public RIPFixup {
6608 public:
6609  JumpTableRIPFixup(CodeGeneratorX86& codegen, HX86PackedSwitch* switch_instr)
6610      : RIPFixup(codegen, static_cast<size_t>(-1)), switch_instr_(switch_instr) {}
6611
6612  void CreateJumpTable() {
6613    X86Assembler* assembler = codegen_->GetAssembler();
6614
6615    // Ensure that the reference to the jump table has the correct offset.
6616    const int32_t offset_in_constant_table = assembler->ConstantAreaSize();
6617    SetOffset(offset_in_constant_table);
6618
6619    // The label values in the jump table are computed relative to the
6620    // instruction addressing the constant area.
6621    const int32_t relative_offset = codegen_->GetMethodAddressOffset();
6622
6623    // Populate the jump table with the correct values for the jump table.
6624    int32_t num_entries = switch_instr_->GetNumEntries();
6625    HBasicBlock* block = switch_instr_->GetBlock();
6626    const ArenaVector<HBasicBlock*>& successors = block->GetSuccessors();
6627    // The value that we want is the target offset - the position of the table.
6628    for (int32_t i = 0; i < num_entries; i++) {
6629      HBasicBlock* b = successors[i];
6630      Label* l = codegen_->GetLabelOf(b);
6631      DCHECK(l->IsBound());
6632      int32_t offset_to_block = l->Position() - relative_offset;
6633      assembler->AppendInt32(offset_to_block);
6634    }
6635  }
6636
6637 private:
6638  const HX86PackedSwitch* switch_instr_;
6639};
6640
6641void CodeGeneratorX86::Finalize(CodeAllocator* allocator) {
6642  // Generate the constant area if needed.
6643  X86Assembler* assembler = GetAssembler();
6644  if (!assembler->IsConstantAreaEmpty() || !fixups_to_jump_tables_.empty()) {
6645    // Align to 4 byte boundary to reduce cache misses, as the data is 4 and 8
6646    // byte values.
6647    assembler->Align(4, 0);
6648    constant_area_start_ = assembler->CodeSize();
6649
6650    // Populate any jump tables.
6651    for (auto jump_table : fixups_to_jump_tables_) {
6652      jump_table->CreateJumpTable();
6653    }
6654
6655    // And now add the constant area to the generated code.
6656    assembler->AddConstantArea();
6657  }
6658
6659  // And finish up.
6660  CodeGenerator::Finalize(allocator);
6661}
6662
6663Address CodeGeneratorX86::LiteralDoubleAddress(double v, Register reg) {
6664  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddDouble(v));
6665  return Address(reg, kDummy32BitOffset, fixup);
6666}
6667
6668Address CodeGeneratorX86::LiteralFloatAddress(float v, Register reg) {
6669  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddFloat(v));
6670  return Address(reg, kDummy32BitOffset, fixup);
6671}
6672
6673Address CodeGeneratorX86::LiteralInt32Address(int32_t v, Register reg) {
6674  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt32(v));
6675  return Address(reg, kDummy32BitOffset, fixup);
6676}
6677
6678Address CodeGeneratorX86::LiteralInt64Address(int64_t v, Register reg) {
6679  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt64(v));
6680  return Address(reg, kDummy32BitOffset, fixup);
6681}
6682
6683Address CodeGeneratorX86::LiteralCaseTable(HX86PackedSwitch* switch_instr,
6684                                           Register reg,
6685                                           Register value) {
6686  // Create a fixup to be used to create and address the jump table.
6687  JumpTableRIPFixup* table_fixup =
6688      new (GetGraph()->GetArena()) JumpTableRIPFixup(*this, switch_instr);
6689
6690  // We have to populate the jump tables.
6691  fixups_to_jump_tables_.push_back(table_fixup);
6692
6693  // We want a scaled address, as we are extracting the correct offset from the table.
6694  return Address(reg, value, TIMES_4, kDummy32BitOffset, table_fixup);
6695}
6696
6697// TODO: target as memory.
6698void CodeGeneratorX86::MoveFromReturnRegister(Location target, Primitive::Type type) {
6699  if (!target.IsValid()) {
6700    DCHECK(type == Primitive::kPrimVoid);
6701    return;
6702  }
6703
6704  DCHECK_NE(type, Primitive::kPrimVoid);
6705
6706  Location return_loc = InvokeDexCallingConventionVisitorX86().GetReturnLocation(type);
6707  if (target.Equals(return_loc)) {
6708    return;
6709  }
6710
6711  // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged
6712  //       with the else branch.
6713  if (type == Primitive::kPrimLong) {
6714    HParallelMove parallel_move(GetGraph()->GetArena());
6715    parallel_move.AddMove(return_loc.ToLow(), target.ToLow(), Primitive::kPrimInt, nullptr);
6716    parallel_move.AddMove(return_loc.ToHigh(), target.ToHigh(), Primitive::kPrimInt, nullptr);
6717    GetMoveResolver()->EmitNativeCode(&parallel_move);
6718  } else {
6719    // Let the parallel move resolver take care of all of this.
6720    HParallelMove parallel_move(GetGraph()->GetArena());
6721    parallel_move.AddMove(return_loc, target, type, nullptr);
6722    GetMoveResolver()->EmitNativeCode(&parallel_move);
6723  }
6724}
6725
6726#undef __
6727
6728}  // namespace x86
6729}  // namespace art
6730