code_generator_x86.cc revision 22c4922c6b31e154a6814c4abe9015d9ba156911 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_x86.h"
18
19#include "art_method.h"
20#include "code_generator_utils.h"
21#include "compiled_method.h"
22#include "entrypoints/quick/quick_entrypoints.h"
23#include "entrypoints/quick/quick_entrypoints_enum.h"
24#include "gc/accounting/card_table.h"
25#include "intrinsics.h"
26#include "intrinsics_x86.h"
27#include "mirror/array-inl.h"
28#include "mirror/class-inl.h"
29#include "thread.h"
30#include "utils/assembler.h"
31#include "utils/stack_checks.h"
32#include "utils/x86/assembler_x86.h"
33#include "utils/x86/managed_register_x86.h"
34
35namespace art {
36
37template<class MirrorType>
38class GcRoot;
39
40namespace x86 {
41
42static constexpr int kCurrentMethodStackOffset = 0;
43static constexpr Register kMethodRegisterArgument = EAX;
44static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI };
45
46static constexpr int kC2ConditionMask = 0x400;
47
48static constexpr int kFakeReturnRegister = Register(8);
49
50#define __ down_cast<X86Assembler*>(codegen->GetAssembler())->
51#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kX86WordSize, x).Int32Value()
52
53class NullCheckSlowPathX86 : public SlowPathCode {
54 public:
55  explicit NullCheckSlowPathX86(HNullCheck* instruction) : SlowPathCode(instruction) {}
56
57  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
58    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
59    __ Bind(GetEntryLabel());
60    if (instruction_->CanThrowIntoCatchBlock()) {
61      // Live registers will be restored in the catch block if caught.
62      SaveLiveRegisters(codegen, instruction_->GetLocations());
63    }
64    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowNullPointer),
65                               instruction_,
66                               instruction_->GetDexPc(),
67                               this);
68    CheckEntrypointTypes<kQuickThrowNullPointer, void, void>();
69  }
70
71  bool IsFatal() const OVERRIDE { return true; }
72
73  const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathX86"; }
74
75 private:
76  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
77};
78
79class DivZeroCheckSlowPathX86 : public SlowPathCode {
80 public:
81  explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : SlowPathCode(instruction) {}
82
83  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
84    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
85    __ Bind(GetEntryLabel());
86    if (instruction_->CanThrowIntoCatchBlock()) {
87      // Live registers will be restored in the catch block if caught.
88      SaveLiveRegisters(codegen, instruction_->GetLocations());
89    }
90    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowDivZero),
91                               instruction_,
92                               instruction_->GetDexPc(),
93                               this);
94    CheckEntrypointTypes<kQuickThrowDivZero, void, void>();
95  }
96
97  bool IsFatal() const OVERRIDE { return true; }
98
99  const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathX86"; }
100
101 private:
102  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
103};
104
105class DivRemMinusOneSlowPathX86 : public SlowPathCode {
106 public:
107  DivRemMinusOneSlowPathX86(HInstruction* instruction, Register reg, bool is_div)
108      : SlowPathCode(instruction), reg_(reg), is_div_(is_div) {}
109
110  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
111    __ Bind(GetEntryLabel());
112    if (is_div_) {
113      __ negl(reg_);
114    } else {
115      __ movl(reg_, Immediate(0));
116    }
117    __ jmp(GetExitLabel());
118  }
119
120  const char* GetDescription() const OVERRIDE { return "DivRemMinusOneSlowPathX86"; }
121
122 private:
123  Register reg_;
124  bool is_div_;
125  DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
126};
127
128class BoundsCheckSlowPathX86 : public SlowPathCode {
129 public:
130  explicit BoundsCheckSlowPathX86(HBoundsCheck* instruction) : SlowPathCode(instruction) {}
131
132  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
133    LocationSummary* locations = instruction_->GetLocations();
134    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
135    __ Bind(GetEntryLabel());
136    // We're moving two locations to locations that could overlap, so we need a parallel
137    // move resolver.
138    if (instruction_->CanThrowIntoCatchBlock()) {
139      // Live registers will be restored in the catch block if caught.
140      SaveLiveRegisters(codegen, instruction_->GetLocations());
141    }
142    InvokeRuntimeCallingConvention calling_convention;
143    x86_codegen->EmitParallelMoves(
144        locations->InAt(0),
145        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
146        Primitive::kPrimInt,
147        locations->InAt(1),
148        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
149        Primitive::kPrimInt);
150    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowArrayBounds),
151                               instruction_,
152                               instruction_->GetDexPc(),
153                               this);
154    CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>();
155  }
156
157  bool IsFatal() const OVERRIDE { return true; }
158
159  const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathX86"; }
160
161 private:
162  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
163};
164
165class SuspendCheckSlowPathX86 : public SlowPathCode {
166 public:
167  SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
168      : SlowPathCode(instruction), successor_(successor) {}
169
170  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
171    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
172    __ Bind(GetEntryLabel());
173    SaveLiveRegisters(codegen, instruction_->GetLocations());
174    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pTestSuspend),
175                               instruction_,
176                               instruction_->GetDexPc(),
177                               this);
178    CheckEntrypointTypes<kQuickTestSuspend, void, void>();
179    RestoreLiveRegisters(codegen, instruction_->GetLocations());
180    if (successor_ == nullptr) {
181      __ jmp(GetReturnLabel());
182    } else {
183      __ jmp(x86_codegen->GetLabelOf(successor_));
184    }
185  }
186
187  Label* GetReturnLabel() {
188    DCHECK(successor_ == nullptr);
189    return &return_label_;
190  }
191
192  HBasicBlock* GetSuccessor() const {
193    return successor_;
194  }
195
196  const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathX86"; }
197
198 private:
199  HBasicBlock* const successor_;
200  Label return_label_;
201
202  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
203};
204
205class LoadStringSlowPathX86 : public SlowPathCode {
206 public:
207  explicit LoadStringSlowPathX86(HLoadString* instruction): SlowPathCode(instruction) {}
208
209  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
210    LocationSummary* locations = instruction_->GetLocations();
211    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
212
213    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
214    __ Bind(GetEntryLabel());
215    SaveLiveRegisters(codegen, locations);
216
217    InvokeRuntimeCallingConvention calling_convention;
218    const uint32_t string_index = instruction_->AsLoadString()->GetStringIndex();
219    __ movl(calling_convention.GetRegisterAt(0), Immediate(string_index));
220    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pResolveString),
221                               instruction_,
222                               instruction_->GetDexPc(),
223                               this);
224    CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
225    x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
226    RestoreLiveRegisters(codegen, locations);
227
228    __ jmp(GetExitLabel());
229  }
230
231  const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathX86"; }
232
233 private:
234  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
235};
236
237class LoadClassSlowPathX86 : public SlowPathCode {
238 public:
239  LoadClassSlowPathX86(HLoadClass* cls,
240                       HInstruction* at,
241                       uint32_t dex_pc,
242                       bool do_clinit)
243      : SlowPathCode(at), cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
244    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
245  }
246
247  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
248    LocationSummary* locations = at_->GetLocations();
249    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
250    __ Bind(GetEntryLabel());
251    SaveLiveRegisters(codegen, locations);
252
253    InvokeRuntimeCallingConvention calling_convention;
254    __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
255    x86_codegen->InvokeRuntime(do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
256                                          : QUICK_ENTRY_POINT(pInitializeType),
257                               at_, dex_pc_, this);
258    if (do_clinit_) {
259      CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>();
260    } else {
261      CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>();
262    }
263
264    // Move the class to the desired location.
265    Location out = locations->Out();
266    if (out.IsValid()) {
267      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
268      x86_codegen->Move32(out, Location::RegisterLocation(EAX));
269    }
270
271    RestoreLiveRegisters(codegen, locations);
272    __ jmp(GetExitLabel());
273  }
274
275  const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathX86"; }
276
277 private:
278  // The class this slow path will load.
279  HLoadClass* const cls_;
280
281  // The instruction where this slow path is happening.
282  // (Might be the load class or an initialization check).
283  HInstruction* const at_;
284
285  // The dex PC of `at_`.
286  const uint32_t dex_pc_;
287
288  // Whether to initialize the class.
289  const bool do_clinit_;
290
291  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
292};
293
294class TypeCheckSlowPathX86 : public SlowPathCode {
295 public:
296  TypeCheckSlowPathX86(HInstruction* instruction, bool is_fatal)
297      : SlowPathCode(instruction), is_fatal_(is_fatal) {}
298
299  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
300    LocationSummary* locations = instruction_->GetLocations();
301    Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0)
302                                                        : locations->Out();
303    DCHECK(instruction_->IsCheckCast()
304           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
305
306    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
307    __ Bind(GetEntryLabel());
308
309    if (!is_fatal_) {
310      SaveLiveRegisters(codegen, locations);
311    }
312
313    // We're moving two locations to locations that could overlap, so we need a parallel
314    // move resolver.
315    InvokeRuntimeCallingConvention calling_convention;
316    x86_codegen->EmitParallelMoves(
317        locations->InAt(1),
318        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
319        Primitive::kPrimNot,
320        object_class,
321        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
322        Primitive::kPrimNot);
323
324    if (instruction_->IsInstanceOf()) {
325      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
326                                 instruction_,
327                                 instruction_->GetDexPc(),
328                                 this);
329      CheckEntrypointTypes<
330          kQuickInstanceofNonTrivial, uint32_t, const mirror::Class*, const mirror::Class*>();
331    } else {
332      DCHECK(instruction_->IsCheckCast());
333      x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
334                                 instruction_,
335                                 instruction_->GetDexPc(),
336                                 this);
337      CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
338    }
339
340    if (!is_fatal_) {
341      if (instruction_->IsInstanceOf()) {
342        x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
343      }
344      RestoreLiveRegisters(codegen, locations);
345
346      __ jmp(GetExitLabel());
347    }
348  }
349
350  const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathX86"; }
351  bool IsFatal() const OVERRIDE { return is_fatal_; }
352
353 private:
354  const bool is_fatal_;
355
356  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
357};
358
359class DeoptimizationSlowPathX86 : public SlowPathCode {
360 public:
361  explicit DeoptimizationSlowPathX86(HDeoptimize* instruction)
362    : SlowPathCode(instruction) {}
363
364  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
365    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
366    __ Bind(GetEntryLabel());
367    SaveLiveRegisters(codegen, instruction_->GetLocations());
368    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize),
369                               instruction_,
370                               instruction_->GetDexPc(),
371                               this);
372    CheckEntrypointTypes<kQuickDeoptimize, void, void>();
373  }
374
375  const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathX86"; }
376
377 private:
378  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86);
379};
380
381class ArraySetSlowPathX86 : public SlowPathCode {
382 public:
383  explicit ArraySetSlowPathX86(HInstruction* instruction) : SlowPathCode(instruction) {}
384
385  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
386    LocationSummary* locations = instruction_->GetLocations();
387    __ Bind(GetEntryLabel());
388    SaveLiveRegisters(codegen, locations);
389
390    InvokeRuntimeCallingConvention calling_convention;
391    HParallelMove parallel_move(codegen->GetGraph()->GetArena());
392    parallel_move.AddMove(
393        locations->InAt(0),
394        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
395        Primitive::kPrimNot,
396        nullptr);
397    parallel_move.AddMove(
398        locations->InAt(1),
399        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
400        Primitive::kPrimInt,
401        nullptr);
402    parallel_move.AddMove(
403        locations->InAt(2),
404        Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
405        Primitive::kPrimNot,
406        nullptr);
407    codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
408
409    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
410    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
411                               instruction_,
412                               instruction_->GetDexPc(),
413                               this);
414    CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>();
415    RestoreLiveRegisters(codegen, locations);
416    __ jmp(GetExitLabel());
417  }
418
419  const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathX86"; }
420
421 private:
422  DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathX86);
423};
424
425// Slow path marking an object during a read barrier.
426class ReadBarrierMarkSlowPathX86 : public SlowPathCode {
427 public:
428  ReadBarrierMarkSlowPathX86(HInstruction* instruction, Location out, Location obj)
429      : SlowPathCode(instruction), out_(out), obj_(obj) {
430    DCHECK(kEmitCompilerReadBarrier);
431  }
432
433  const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathX86"; }
434
435  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
436    LocationSummary* locations = instruction_->GetLocations();
437    Register reg_out = out_.AsRegister<Register>();
438    DCHECK(locations->CanCall());
439    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
440    DCHECK(instruction_->IsInstanceFieldGet() ||
441           instruction_->IsStaticFieldGet() ||
442           instruction_->IsArrayGet() ||
443           instruction_->IsLoadClass() ||
444           instruction_->IsLoadString() ||
445           instruction_->IsInstanceOf() ||
446           instruction_->IsCheckCast())
447        << "Unexpected instruction in read barrier marking slow path: "
448        << instruction_->DebugName();
449
450    __ Bind(GetEntryLabel());
451    SaveLiveRegisters(codegen, locations);
452
453    InvokeRuntimeCallingConvention calling_convention;
454    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
455    x86_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), obj_);
456    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierMark),
457                               instruction_,
458                               instruction_->GetDexPc(),
459                               this);
460    CheckEntrypointTypes<kQuickReadBarrierMark, mirror::Object*, mirror::Object*>();
461    x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
462
463    RestoreLiveRegisters(codegen, locations);
464    __ jmp(GetExitLabel());
465  }
466
467 private:
468  const Location out_;
469  const Location obj_;
470
471  DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathX86);
472};
473
474// Slow path generating a read barrier for a heap reference.
475class ReadBarrierForHeapReferenceSlowPathX86 : public SlowPathCode {
476 public:
477  ReadBarrierForHeapReferenceSlowPathX86(HInstruction* instruction,
478                                         Location out,
479                                         Location ref,
480                                         Location obj,
481                                         uint32_t offset,
482                                         Location index)
483      : SlowPathCode(instruction),
484        out_(out),
485        ref_(ref),
486        obj_(obj),
487        offset_(offset),
488        index_(index) {
489    DCHECK(kEmitCompilerReadBarrier);
490    // If `obj` is equal to `out` or `ref`, it means the initial object
491    // has been overwritten by (or after) the heap object reference load
492    // to be instrumented, e.g.:
493    //
494    //   __ movl(out, Address(out, offset));
495    //   codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset);
496    //
497    // In that case, we have lost the information about the original
498    // object, and the emitted read barrier cannot work properly.
499    DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out;
500    DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref;
501  }
502
503  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
504    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
505    LocationSummary* locations = instruction_->GetLocations();
506    Register reg_out = out_.AsRegister<Register>();
507    DCHECK(locations->CanCall());
508    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
509    DCHECK(!instruction_->IsInvoke() ||
510           (instruction_->IsInvokeStaticOrDirect() &&
511            instruction_->GetLocations()->Intrinsified()))
512        << "Unexpected instruction in read barrier for heap reference slow path: "
513        << instruction_->DebugName();
514
515    __ Bind(GetEntryLabel());
516    SaveLiveRegisters(codegen, locations);
517
518    // We may have to change the index's value, but as `index_` is a
519    // constant member (like other "inputs" of this slow path),
520    // introduce a copy of it, `index`.
521    Location index = index_;
522    if (index_.IsValid()) {
523      // Handle `index_` for HArrayGet and intrinsic UnsafeGetObject.
524      if (instruction_->IsArrayGet()) {
525        // Compute the actual memory offset and store it in `index`.
526        Register index_reg = index_.AsRegister<Register>();
527        DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg));
528        if (codegen->IsCoreCalleeSaveRegister(index_reg)) {
529          // We are about to change the value of `index_reg` (see the
530          // calls to art::x86::X86Assembler::shll and
531          // art::x86::X86Assembler::AddImmediate below), but it has
532          // not been saved by the previous call to
533          // art::SlowPathCode::SaveLiveRegisters, as it is a
534          // callee-save register --
535          // art::SlowPathCode::SaveLiveRegisters does not consider
536          // callee-save registers, as it has been designed with the
537          // assumption that callee-save registers are supposed to be
538          // handled by the called function.  So, as a callee-save
539          // register, `index_reg` _would_ eventually be saved onto
540          // the stack, but it would be too late: we would have
541          // changed its value earlier.  Therefore, we manually save
542          // it here into another freely available register,
543          // `free_reg`, chosen of course among the caller-save
544          // registers (as a callee-save `free_reg` register would
545          // exhibit the same problem).
546          //
547          // Note we could have requested a temporary register from
548          // the register allocator instead; but we prefer not to, as
549          // this is a slow path, and we know we can find a
550          // caller-save register that is available.
551          Register free_reg = FindAvailableCallerSaveRegister(codegen);
552          __ movl(free_reg, index_reg);
553          index_reg = free_reg;
554          index = Location::RegisterLocation(index_reg);
555        } else {
556          // The initial register stored in `index_` has already been
557          // saved in the call to art::SlowPathCode::SaveLiveRegisters
558          // (as it is not a callee-save register), so we can freely
559          // use it.
560        }
561        // Shifting the index value contained in `index_reg` by the scale
562        // factor (2) cannot overflow in practice, as the runtime is
563        // unable to allocate object arrays with a size larger than
564        // 2^26 - 1 (that is, 2^28 - 4 bytes).
565        __ shll(index_reg, Immediate(TIMES_4));
566        static_assert(
567            sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
568            "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
569        __ AddImmediate(index_reg, Immediate(offset_));
570      } else {
571        DCHECK(instruction_->IsInvoke());
572        DCHECK(instruction_->GetLocations()->Intrinsified());
573        DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) ||
574               (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile))
575            << instruction_->AsInvoke()->GetIntrinsic();
576        DCHECK_EQ(offset_, 0U);
577        DCHECK(index_.IsRegisterPair());
578        // UnsafeGet's offset location is a register pair, the low
579        // part contains the correct offset.
580        index = index_.ToLow();
581      }
582    }
583
584    // We're moving two or three locations to locations that could
585    // overlap, so we need a parallel move resolver.
586    InvokeRuntimeCallingConvention calling_convention;
587    HParallelMove parallel_move(codegen->GetGraph()->GetArena());
588    parallel_move.AddMove(ref_,
589                          Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
590                          Primitive::kPrimNot,
591                          nullptr);
592    parallel_move.AddMove(obj_,
593                          Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
594                          Primitive::kPrimNot,
595                          nullptr);
596    if (index.IsValid()) {
597      parallel_move.AddMove(index,
598                            Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
599                            Primitive::kPrimInt,
600                            nullptr);
601      codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
602    } else {
603      codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
604      __ movl(calling_convention.GetRegisterAt(2), Immediate(offset_));
605    }
606    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierSlow),
607                               instruction_,
608                               instruction_->GetDexPc(),
609                               this);
610    CheckEntrypointTypes<
611        kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>();
612    x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
613
614    RestoreLiveRegisters(codegen, locations);
615    __ jmp(GetExitLabel());
616  }
617
618  const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathX86"; }
619
620 private:
621  Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) {
622    size_t ref = static_cast<int>(ref_.AsRegister<Register>());
623    size_t obj = static_cast<int>(obj_.AsRegister<Register>());
624    for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
625      if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) {
626        return static_cast<Register>(i);
627      }
628    }
629    // We shall never fail to find a free caller-save register, as
630    // there are more than two core caller-save registers on x86
631    // (meaning it is possible to find one which is different from
632    // `ref` and `obj`).
633    DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u);
634    LOG(FATAL) << "Could not find a free caller-save register";
635    UNREACHABLE();
636  }
637
638  const Location out_;
639  const Location ref_;
640  const Location obj_;
641  const uint32_t offset_;
642  // An additional location containing an index to an array.
643  // Only used for HArrayGet and the UnsafeGetObject &
644  // UnsafeGetObjectVolatile intrinsics.
645  const Location index_;
646
647  DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathX86);
648};
649
650// Slow path generating a read barrier for a GC root.
651class ReadBarrierForRootSlowPathX86 : public SlowPathCode {
652 public:
653  ReadBarrierForRootSlowPathX86(HInstruction* instruction, Location out, Location root)
654      : SlowPathCode(instruction), out_(out), root_(root) {
655    DCHECK(kEmitCompilerReadBarrier);
656  }
657
658  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
659    LocationSummary* locations = instruction_->GetLocations();
660    Register reg_out = out_.AsRegister<Register>();
661    DCHECK(locations->CanCall());
662    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
663    DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString())
664        << "Unexpected instruction in read barrier for GC root slow path: "
665        << instruction_->DebugName();
666
667    __ Bind(GetEntryLabel());
668    SaveLiveRegisters(codegen, locations);
669
670    InvokeRuntimeCallingConvention calling_convention;
671    CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
672    x86_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_);
673    x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierForRootSlow),
674                               instruction_,
675                               instruction_->GetDexPc(),
676                               this);
677    CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>();
678    x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
679
680    RestoreLiveRegisters(codegen, locations);
681    __ jmp(GetExitLabel());
682  }
683
684  const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathX86"; }
685
686 private:
687  const Location out_;
688  const Location root_;
689
690  DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathX86);
691};
692
693#undef __
694#define __ down_cast<X86Assembler*>(GetAssembler())->
695
696inline Condition X86Condition(IfCondition cond) {
697  switch (cond) {
698    case kCondEQ: return kEqual;
699    case kCondNE: return kNotEqual;
700    case kCondLT: return kLess;
701    case kCondLE: return kLessEqual;
702    case kCondGT: return kGreater;
703    case kCondGE: return kGreaterEqual;
704    case kCondB:  return kBelow;
705    case kCondBE: return kBelowEqual;
706    case kCondA:  return kAbove;
707    case kCondAE: return kAboveEqual;
708  }
709  LOG(FATAL) << "Unreachable";
710  UNREACHABLE();
711}
712
713// Maps signed condition to unsigned condition and FP condition to x86 name.
714inline Condition X86UnsignedOrFPCondition(IfCondition cond) {
715  switch (cond) {
716    case kCondEQ: return kEqual;
717    case kCondNE: return kNotEqual;
718    // Signed to unsigned, and FP to x86 name.
719    case kCondLT: return kBelow;
720    case kCondLE: return kBelowEqual;
721    case kCondGT: return kAbove;
722    case kCondGE: return kAboveEqual;
723    // Unsigned remain unchanged.
724    case kCondB:  return kBelow;
725    case kCondBE: return kBelowEqual;
726    case kCondA:  return kAbove;
727    case kCondAE: return kAboveEqual;
728  }
729  LOG(FATAL) << "Unreachable";
730  UNREACHABLE();
731}
732
733void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
734  stream << Register(reg);
735}
736
737void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
738  stream << XmmRegister(reg);
739}
740
741size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
742  __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
743  return kX86WordSize;
744}
745
746size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
747  __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
748  return kX86WordSize;
749}
750
751size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
752  __ movsd(Address(ESP, stack_index), XmmRegister(reg_id));
753  return GetFloatingPointSpillSlotSize();
754}
755
756size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
757  __ movsd(XmmRegister(reg_id), Address(ESP, stack_index));
758  return GetFloatingPointSpillSlotSize();
759}
760
761void CodeGeneratorX86::InvokeRuntime(QuickEntrypointEnum entrypoint,
762                                     HInstruction* instruction,
763                                     uint32_t dex_pc,
764                                     SlowPathCode* slow_path) {
765  InvokeRuntime(GetThreadOffset<kX86WordSize>(entrypoint).Int32Value(),
766                instruction,
767                dex_pc,
768                slow_path);
769}
770
771void CodeGeneratorX86::InvokeRuntime(int32_t entry_point_offset,
772                                     HInstruction* instruction,
773                                     uint32_t dex_pc,
774                                     SlowPathCode* slow_path) {
775  ValidateInvokeRuntime(instruction, slow_path);
776  __ fs()->call(Address::Absolute(entry_point_offset));
777  RecordPcInfo(instruction, dex_pc, slow_path);
778}
779
780CodeGeneratorX86::CodeGeneratorX86(HGraph* graph,
781                                   const X86InstructionSetFeatures& isa_features,
782                                   const CompilerOptions& compiler_options,
783                                   OptimizingCompilerStats* stats)
784    : CodeGenerator(graph,
785                    kNumberOfCpuRegisters,
786                    kNumberOfXmmRegisters,
787                    kNumberOfRegisterPairs,
788                    ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
789                                        arraysize(kCoreCalleeSaves))
790                        | (1 << kFakeReturnRegister),
791                    0,
792                    compiler_options,
793                    stats),
794      block_labels_(nullptr),
795      location_builder_(graph, this),
796      instruction_visitor_(graph, this),
797      move_resolver_(graph->GetArena(), this),
798      isa_features_(isa_features),
799      method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
800      relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
801      pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
802      fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
803  // Use a fake return address register to mimic Quick.
804  AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
805}
806
807void CodeGeneratorX86::SetupBlockedRegisters() const {
808  // Don't allocate the dalvik style register pair passing.
809  blocked_register_pairs_[ECX_EDX] = true;
810
811  // Stack register is always reserved.
812  blocked_core_registers_[ESP] = true;
813
814  UpdateBlockedPairRegisters();
815}
816
817void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
818  for (int i = 0; i < kNumberOfRegisterPairs; i++) {
819    X86ManagedRegister current =
820        X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
821    if (blocked_core_registers_[current.AsRegisterPairLow()]
822        || blocked_core_registers_[current.AsRegisterPairHigh()]) {
823      blocked_register_pairs_[i] = true;
824    }
825  }
826}
827
828InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
829      : InstructionCodeGenerator(graph, codegen),
830        assembler_(codegen->GetAssembler()),
831        codegen_(codegen) {}
832
833static dwarf::Reg DWARFReg(Register reg) {
834  return dwarf::Reg::X86Core(static_cast<int>(reg));
835}
836
837void CodeGeneratorX86::GenerateFrameEntry() {
838  __ cfi().SetCurrentCFAOffset(kX86WordSize);  // return address
839  __ Bind(&frame_entry_label_);
840  bool skip_overflow_check =
841      IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
842  DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
843
844  if (!skip_overflow_check) {
845    __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
846    RecordPcInfo(nullptr, 0);
847  }
848
849  if (HasEmptyFrame()) {
850    return;
851  }
852
853  for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
854    Register reg = kCoreCalleeSaves[i];
855    if (allocated_registers_.ContainsCoreRegister(reg)) {
856      __ pushl(reg);
857      __ cfi().AdjustCFAOffset(kX86WordSize);
858      __ cfi().RelOffset(DWARFReg(reg), 0);
859    }
860  }
861
862  int adjust = GetFrameSize() - FrameEntrySpillSize();
863  __ subl(ESP, Immediate(adjust));
864  __ cfi().AdjustCFAOffset(adjust);
865  __ movl(Address(ESP, kCurrentMethodStackOffset), kMethodRegisterArgument);
866}
867
868void CodeGeneratorX86::GenerateFrameExit() {
869  __ cfi().RememberState();
870  if (!HasEmptyFrame()) {
871    int adjust = GetFrameSize() - FrameEntrySpillSize();
872    __ addl(ESP, Immediate(adjust));
873    __ cfi().AdjustCFAOffset(-adjust);
874
875    for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
876      Register reg = kCoreCalleeSaves[i];
877      if (allocated_registers_.ContainsCoreRegister(reg)) {
878        __ popl(reg);
879        __ cfi().AdjustCFAOffset(-static_cast<int>(kX86WordSize));
880        __ cfi().Restore(DWARFReg(reg));
881      }
882    }
883  }
884  __ ret();
885  __ cfi().RestoreState();
886  __ cfi().DefCFAOffset(GetFrameSize());
887}
888
889void CodeGeneratorX86::Bind(HBasicBlock* block) {
890  __ Bind(GetLabelOf(block));
891}
892
893Location CodeGeneratorX86::GetStackLocation(HLoadLocal* load) const {
894  switch (load->GetType()) {
895    case Primitive::kPrimLong:
896    case Primitive::kPrimDouble:
897      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
898
899    case Primitive::kPrimInt:
900    case Primitive::kPrimNot:
901    case Primitive::kPrimFloat:
902      return Location::StackSlot(GetStackSlot(load->GetLocal()));
903
904    case Primitive::kPrimBoolean:
905    case Primitive::kPrimByte:
906    case Primitive::kPrimChar:
907    case Primitive::kPrimShort:
908    case Primitive::kPrimVoid:
909      LOG(FATAL) << "Unexpected type " << load->GetType();
910      UNREACHABLE();
911  }
912
913  LOG(FATAL) << "Unreachable";
914  UNREACHABLE();
915}
916
917Location InvokeDexCallingConventionVisitorX86::GetReturnLocation(Primitive::Type type) const {
918  switch (type) {
919    case Primitive::kPrimBoolean:
920    case Primitive::kPrimByte:
921    case Primitive::kPrimChar:
922    case Primitive::kPrimShort:
923    case Primitive::kPrimInt:
924    case Primitive::kPrimNot:
925      return Location::RegisterLocation(EAX);
926
927    case Primitive::kPrimLong:
928      return Location::RegisterPairLocation(EAX, EDX);
929
930    case Primitive::kPrimVoid:
931      return Location::NoLocation();
932
933    case Primitive::kPrimDouble:
934    case Primitive::kPrimFloat:
935      return Location::FpuRegisterLocation(XMM0);
936  }
937
938  UNREACHABLE();
939}
940
941Location InvokeDexCallingConventionVisitorX86::GetMethodLocation() const {
942  return Location::RegisterLocation(kMethodRegisterArgument);
943}
944
945Location InvokeDexCallingConventionVisitorX86::GetNextLocation(Primitive::Type type) {
946  switch (type) {
947    case Primitive::kPrimBoolean:
948    case Primitive::kPrimByte:
949    case Primitive::kPrimChar:
950    case Primitive::kPrimShort:
951    case Primitive::kPrimInt:
952    case Primitive::kPrimNot: {
953      uint32_t index = gp_index_++;
954      stack_index_++;
955      if (index < calling_convention.GetNumberOfRegisters()) {
956        return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
957      } else {
958        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
959      }
960    }
961
962    case Primitive::kPrimLong: {
963      uint32_t index = gp_index_;
964      gp_index_ += 2;
965      stack_index_ += 2;
966      if (index + 1 < calling_convention.GetNumberOfRegisters()) {
967        X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
968            calling_convention.GetRegisterPairAt(index));
969        return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
970      } else {
971        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
972      }
973    }
974
975    case Primitive::kPrimFloat: {
976      uint32_t index = float_index_++;
977      stack_index_++;
978      if (index < calling_convention.GetNumberOfFpuRegisters()) {
979        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
980      } else {
981        return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
982      }
983    }
984
985    case Primitive::kPrimDouble: {
986      uint32_t index = float_index_++;
987      stack_index_ += 2;
988      if (index < calling_convention.GetNumberOfFpuRegisters()) {
989        return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
990      } else {
991        return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
992      }
993    }
994
995    case Primitive::kPrimVoid:
996      LOG(FATAL) << "Unexpected parameter type " << type;
997      break;
998  }
999  return Location::NoLocation();
1000}
1001
1002void CodeGeneratorX86::Move32(Location destination, Location source) {
1003  if (source.Equals(destination)) {
1004    return;
1005  }
1006  if (destination.IsRegister()) {
1007    if (source.IsRegister()) {
1008      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
1009    } else if (source.IsFpuRegister()) {
1010      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
1011    } else {
1012      DCHECK(source.IsStackSlot());
1013      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
1014    }
1015  } else if (destination.IsFpuRegister()) {
1016    if (source.IsRegister()) {
1017      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
1018    } else if (source.IsFpuRegister()) {
1019      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
1020    } else {
1021      DCHECK(source.IsStackSlot());
1022      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
1023    }
1024  } else {
1025    DCHECK(destination.IsStackSlot()) << destination;
1026    if (source.IsRegister()) {
1027      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
1028    } else if (source.IsFpuRegister()) {
1029      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
1030    } else if (source.IsConstant()) {
1031      HConstant* constant = source.GetConstant();
1032      int32_t value = GetInt32ValueOf(constant);
1033      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
1034    } else {
1035      DCHECK(source.IsStackSlot());
1036      __ pushl(Address(ESP, source.GetStackIndex()));
1037      __ popl(Address(ESP, destination.GetStackIndex()));
1038    }
1039  }
1040}
1041
1042void CodeGeneratorX86::Move64(Location destination, Location source) {
1043  if (source.Equals(destination)) {
1044    return;
1045  }
1046  if (destination.IsRegisterPair()) {
1047    if (source.IsRegisterPair()) {
1048      EmitParallelMoves(
1049          Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
1050          Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
1051          Primitive::kPrimInt,
1052          Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
1053          Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
1054          Primitive::kPrimInt);
1055    } else if (source.IsFpuRegister()) {
1056      XmmRegister src_reg = source.AsFpuRegister<XmmRegister>();
1057      __ movd(destination.AsRegisterPairLow<Register>(), src_reg);
1058      __ psrlq(src_reg, Immediate(32));
1059      __ movd(destination.AsRegisterPairHigh<Register>(), src_reg);
1060    } else {
1061      // No conflict possible, so just do the moves.
1062      DCHECK(source.IsDoubleStackSlot());
1063      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
1064      __ movl(destination.AsRegisterPairHigh<Register>(),
1065              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
1066    }
1067  } else if (destination.IsFpuRegister()) {
1068    if (source.IsFpuRegister()) {
1069      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
1070    } else if (source.IsDoubleStackSlot()) {
1071      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
1072    } else if (source.IsRegisterPair()) {
1073      size_t elem_size = Primitive::ComponentSize(Primitive::kPrimInt);
1074      // Create stack space for 2 elements.
1075      __ subl(ESP, Immediate(2 * elem_size));
1076      __ movl(Address(ESP, 0), source.AsRegisterPairLow<Register>());
1077      __ movl(Address(ESP, elem_size), source.AsRegisterPairHigh<Register>());
1078      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
1079      // And remove the temporary stack space we allocated.
1080      __ addl(ESP, Immediate(2 * elem_size));
1081    } else {
1082      LOG(FATAL) << "Unimplemented";
1083    }
1084  } else {
1085    DCHECK(destination.IsDoubleStackSlot()) << destination;
1086    if (source.IsRegisterPair()) {
1087      // No conflict possible, so just do the moves.
1088      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
1089      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
1090              source.AsRegisterPairHigh<Register>());
1091    } else if (source.IsFpuRegister()) {
1092      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
1093    } else if (source.IsConstant()) {
1094      HConstant* constant = source.GetConstant();
1095      int64_t value;
1096      if (constant->IsLongConstant()) {
1097        value = constant->AsLongConstant()->GetValue();
1098      } else {
1099        DCHECK(constant->IsDoubleConstant());
1100        value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
1101      }
1102      __ movl(Address(ESP, destination.GetStackIndex()), Immediate(Low32Bits(value)));
1103      __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value)));
1104    } else {
1105      DCHECK(source.IsDoubleStackSlot()) << source;
1106      EmitParallelMoves(
1107          Location::StackSlot(source.GetStackIndex()),
1108          Location::StackSlot(destination.GetStackIndex()),
1109          Primitive::kPrimInt,
1110          Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
1111          Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)),
1112          Primitive::kPrimInt);
1113    }
1114  }
1115}
1116
1117void CodeGeneratorX86::MoveConstant(Location location, int32_t value) {
1118  DCHECK(location.IsRegister());
1119  __ movl(location.AsRegister<Register>(), Immediate(value));
1120}
1121
1122void CodeGeneratorX86::MoveLocation(Location dst, Location src, Primitive::Type dst_type) {
1123  HParallelMove move(GetGraph()->GetArena());
1124  if (dst_type == Primitive::kPrimLong && !src.IsConstant() && !src.IsFpuRegister()) {
1125    move.AddMove(src.ToLow(), dst.ToLow(), Primitive::kPrimInt, nullptr);
1126    move.AddMove(src.ToHigh(), dst.ToHigh(), Primitive::kPrimInt, nullptr);
1127  } else {
1128    move.AddMove(src, dst, dst_type, nullptr);
1129  }
1130  GetMoveResolver()->EmitNativeCode(&move);
1131}
1132
1133void CodeGeneratorX86::AddLocationAsTemp(Location location, LocationSummary* locations) {
1134  if (location.IsRegister()) {
1135    locations->AddTemp(location);
1136  } else if (location.IsRegisterPair()) {
1137    locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>()));
1138    locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>()));
1139  } else {
1140    UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location;
1141  }
1142}
1143
1144void InstructionCodeGeneratorX86::HandleGoto(HInstruction* got, HBasicBlock* successor) {
1145  DCHECK(!successor->IsExitBlock());
1146
1147  HBasicBlock* block = got->GetBlock();
1148  HInstruction* previous = got->GetPrevious();
1149
1150  HLoopInformation* info = block->GetLoopInformation();
1151  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
1152    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
1153    return;
1154  }
1155
1156  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
1157    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
1158  }
1159  if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
1160    __ jmp(codegen_->GetLabelOf(successor));
1161  }
1162}
1163
1164void LocationsBuilderX86::VisitGoto(HGoto* got) {
1165  got->SetLocations(nullptr);
1166}
1167
1168void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
1169  HandleGoto(got, got->GetSuccessor());
1170}
1171
1172void LocationsBuilderX86::VisitTryBoundary(HTryBoundary* try_boundary) {
1173  try_boundary->SetLocations(nullptr);
1174}
1175
1176void InstructionCodeGeneratorX86::VisitTryBoundary(HTryBoundary* try_boundary) {
1177  HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
1178  if (!successor->IsExitBlock()) {
1179    HandleGoto(try_boundary, successor);
1180  }
1181}
1182
1183void LocationsBuilderX86::VisitExit(HExit* exit) {
1184  exit->SetLocations(nullptr);
1185}
1186
1187void InstructionCodeGeneratorX86::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
1188}
1189
1190template<class LabelType>
1191void InstructionCodeGeneratorX86::GenerateFPJumps(HCondition* cond,
1192                                                  LabelType* true_label,
1193                                                  LabelType* false_label) {
1194  if (cond->IsFPConditionTrueIfNaN()) {
1195    __ j(kUnordered, true_label);
1196  } else if (cond->IsFPConditionFalseIfNaN()) {
1197    __ j(kUnordered, false_label);
1198  }
1199  __ j(X86UnsignedOrFPCondition(cond->GetCondition()), true_label);
1200}
1201
1202template<class LabelType>
1203void InstructionCodeGeneratorX86::GenerateLongComparesAndJumps(HCondition* cond,
1204                                                               LabelType* true_label,
1205                                                               LabelType* false_label) {
1206  LocationSummary* locations = cond->GetLocations();
1207  Location left = locations->InAt(0);
1208  Location right = locations->InAt(1);
1209  IfCondition if_cond = cond->GetCondition();
1210
1211  Register left_high = left.AsRegisterPairHigh<Register>();
1212  Register left_low = left.AsRegisterPairLow<Register>();
1213  IfCondition true_high_cond = if_cond;
1214  IfCondition false_high_cond = cond->GetOppositeCondition();
1215  Condition final_condition = X86UnsignedOrFPCondition(if_cond);  // unsigned on lower part
1216
1217  // Set the conditions for the test, remembering that == needs to be
1218  // decided using the low words.
1219  switch (if_cond) {
1220    case kCondEQ:
1221    case kCondNE:
1222      // Nothing to do.
1223      break;
1224    case kCondLT:
1225      false_high_cond = kCondGT;
1226      break;
1227    case kCondLE:
1228      true_high_cond = kCondLT;
1229      break;
1230    case kCondGT:
1231      false_high_cond = kCondLT;
1232      break;
1233    case kCondGE:
1234      true_high_cond = kCondGT;
1235      break;
1236    case kCondB:
1237      false_high_cond = kCondA;
1238      break;
1239    case kCondBE:
1240      true_high_cond = kCondB;
1241      break;
1242    case kCondA:
1243      false_high_cond = kCondB;
1244      break;
1245    case kCondAE:
1246      true_high_cond = kCondA;
1247      break;
1248  }
1249
1250  if (right.IsConstant()) {
1251    int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
1252    int32_t val_high = High32Bits(value);
1253    int32_t val_low = Low32Bits(value);
1254
1255    codegen_->Compare32BitValue(left_high, val_high);
1256    if (if_cond == kCondNE) {
1257      __ j(X86Condition(true_high_cond), true_label);
1258    } else if (if_cond == kCondEQ) {
1259      __ j(X86Condition(false_high_cond), false_label);
1260    } else {
1261      __ j(X86Condition(true_high_cond), true_label);
1262      __ j(X86Condition(false_high_cond), false_label);
1263    }
1264    // Must be equal high, so compare the lows.
1265    codegen_->Compare32BitValue(left_low, val_low);
1266  } else if (right.IsRegisterPair()) {
1267    Register right_high = right.AsRegisterPairHigh<Register>();
1268    Register right_low = right.AsRegisterPairLow<Register>();
1269
1270    __ cmpl(left_high, right_high);
1271    if (if_cond == kCondNE) {
1272      __ j(X86Condition(true_high_cond), true_label);
1273    } else if (if_cond == kCondEQ) {
1274      __ j(X86Condition(false_high_cond), false_label);
1275    } else {
1276      __ j(X86Condition(true_high_cond), true_label);
1277      __ j(X86Condition(false_high_cond), false_label);
1278    }
1279    // Must be equal high, so compare the lows.
1280    __ cmpl(left_low, right_low);
1281  } else {
1282    DCHECK(right.IsDoubleStackSlot());
1283    __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
1284    if (if_cond == kCondNE) {
1285      __ j(X86Condition(true_high_cond), true_label);
1286    } else if (if_cond == kCondEQ) {
1287      __ j(X86Condition(false_high_cond), false_label);
1288    } else {
1289      __ j(X86Condition(true_high_cond), true_label);
1290      __ j(X86Condition(false_high_cond), false_label);
1291    }
1292    // Must be equal high, so compare the lows.
1293    __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
1294  }
1295  // The last comparison might be unsigned.
1296  __ j(final_condition, true_label);
1297}
1298
1299void InstructionCodeGeneratorX86::GenerateFPCompare(Location lhs,
1300                                                    Location rhs,
1301                                                    HInstruction* insn,
1302                                                    bool is_double) {
1303  HX86LoadFromConstantTable* const_area = insn->InputAt(1)->AsX86LoadFromConstantTable();
1304  if (is_double) {
1305    if (rhs.IsFpuRegister()) {
1306      __ ucomisd(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1307    } else if (const_area != nullptr) {
1308      DCHECK(const_area->IsEmittedAtUseSite());
1309      __ ucomisd(lhs.AsFpuRegister<XmmRegister>(),
1310                 codegen_->LiteralDoubleAddress(
1311                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
1312                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
1313    } else {
1314      DCHECK(rhs.IsDoubleStackSlot());
1315      __ ucomisd(lhs.AsFpuRegister<XmmRegister>(), Address(ESP, rhs.GetStackIndex()));
1316    }
1317  } else {
1318    if (rhs.IsFpuRegister()) {
1319      __ ucomiss(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1320    } else if (const_area != nullptr) {
1321      DCHECK(const_area->IsEmittedAtUseSite());
1322      __ ucomiss(lhs.AsFpuRegister<XmmRegister>(),
1323                 codegen_->LiteralFloatAddress(
1324                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
1325                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
1326    } else {
1327      DCHECK(rhs.IsStackSlot());
1328      __ ucomiss(lhs.AsFpuRegister<XmmRegister>(), Address(ESP, rhs.GetStackIndex()));
1329    }
1330  }
1331}
1332
1333template<class LabelType>
1334void InstructionCodeGeneratorX86::GenerateCompareTestAndBranch(HCondition* condition,
1335                                                               LabelType* true_target_in,
1336                                                               LabelType* false_target_in) {
1337  // Generated branching requires both targets to be explicit. If either of the
1338  // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead.
1339  LabelType fallthrough_target;
1340  LabelType* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in;
1341  LabelType* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in;
1342
1343  LocationSummary* locations = condition->GetLocations();
1344  Location left = locations->InAt(0);
1345  Location right = locations->InAt(1);
1346
1347  Primitive::Type type = condition->InputAt(0)->GetType();
1348  switch (type) {
1349    case Primitive::kPrimLong:
1350      GenerateLongComparesAndJumps(condition, true_target, false_target);
1351      break;
1352    case Primitive::kPrimFloat:
1353      GenerateFPCompare(left, right, condition, false);
1354      GenerateFPJumps(condition, true_target, false_target);
1355      break;
1356    case Primitive::kPrimDouble:
1357      GenerateFPCompare(left, right, condition, true);
1358      GenerateFPJumps(condition, true_target, false_target);
1359      break;
1360    default:
1361      LOG(FATAL) << "Unexpected compare type " << type;
1362  }
1363
1364  if (false_target != &fallthrough_target) {
1365    __ jmp(false_target);
1366  }
1367
1368  if (fallthrough_target.IsLinked()) {
1369    __ Bind(&fallthrough_target);
1370  }
1371}
1372
1373static bool AreEflagsSetFrom(HInstruction* cond, HInstruction* branch) {
1374  // Moves may affect the eflags register (move zero uses xorl), so the EFLAGS
1375  // are set only strictly before `branch`. We can't use the eflags on long/FP
1376  // conditions if they are materialized due to the complex branching.
1377  return cond->IsCondition() &&
1378         cond->GetNext() == branch &&
1379         cond->InputAt(0)->GetType() != Primitive::kPrimLong &&
1380         !Primitive::IsFloatingPointType(cond->InputAt(0)->GetType());
1381}
1382
1383template<class LabelType>
1384void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instruction,
1385                                                        size_t condition_input_index,
1386                                                        LabelType* true_target,
1387                                                        LabelType* false_target) {
1388  HInstruction* cond = instruction->InputAt(condition_input_index);
1389
1390  if (true_target == nullptr && false_target == nullptr) {
1391    // Nothing to do. The code always falls through.
1392    return;
1393  } else if (cond->IsIntConstant()) {
1394    // Constant condition, statically compared against 1.
1395    if (cond->AsIntConstant()->IsOne()) {
1396      if (true_target != nullptr) {
1397        __ jmp(true_target);
1398      }
1399    } else {
1400      DCHECK(cond->AsIntConstant()->IsZero());
1401      if (false_target != nullptr) {
1402        __ jmp(false_target);
1403      }
1404    }
1405    return;
1406  }
1407
1408  // The following code generates these patterns:
1409  //  (1) true_target == nullptr && false_target != nullptr
1410  //        - opposite condition true => branch to false_target
1411  //  (2) true_target != nullptr && false_target == nullptr
1412  //        - condition true => branch to true_target
1413  //  (3) true_target != nullptr && false_target != nullptr
1414  //        - condition true => branch to true_target
1415  //        - branch to false_target
1416  if (IsBooleanValueOrMaterializedCondition(cond)) {
1417    if (AreEflagsSetFrom(cond, instruction)) {
1418      if (true_target == nullptr) {
1419        __ j(X86Condition(cond->AsCondition()->GetOppositeCondition()), false_target);
1420      } else {
1421        __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
1422      }
1423    } else {
1424      // Materialized condition, compare against 0.
1425      Location lhs = instruction->GetLocations()->InAt(condition_input_index);
1426      if (lhs.IsRegister()) {
1427        __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1428      } else {
1429        __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
1430      }
1431      if (true_target == nullptr) {
1432        __ j(kEqual, false_target);
1433      } else {
1434        __ j(kNotEqual, true_target);
1435      }
1436    }
1437  } else {
1438    // Condition has not been materialized, use its inputs as the comparison and
1439    // its condition as the branch condition.
1440    HCondition* condition = cond->AsCondition();
1441
1442    // If this is a long or FP comparison that has been folded into
1443    // the HCondition, generate the comparison directly.
1444    Primitive::Type type = condition->InputAt(0)->GetType();
1445    if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
1446      GenerateCompareTestAndBranch(condition, true_target, false_target);
1447      return;
1448    }
1449
1450    Location lhs = condition->GetLocations()->InAt(0);
1451    Location rhs = condition->GetLocations()->InAt(1);
1452    // LHS is guaranteed to be in a register (see LocationsBuilderX86::HandleCondition).
1453    if (rhs.IsRegister()) {
1454      __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1455    } else if (rhs.IsConstant()) {
1456      int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1457      codegen_->Compare32BitValue(lhs.AsRegister<Register>(), constant);
1458    } else {
1459      __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1460    }
1461    if (true_target == nullptr) {
1462      __ j(X86Condition(condition->GetOppositeCondition()), false_target);
1463    } else {
1464      __ j(X86Condition(condition->GetCondition()), true_target);
1465    }
1466  }
1467
1468  // If neither branch falls through (case 3), the conditional branch to `true_target`
1469  // was already emitted (case 2) and we need to emit a jump to `false_target`.
1470  if (true_target != nullptr && false_target != nullptr) {
1471    __ jmp(false_target);
1472  }
1473}
1474
1475void LocationsBuilderX86::VisitIf(HIf* if_instr) {
1476  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
1477  if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
1478    locations->SetInAt(0, Location::Any());
1479  }
1480}
1481
1482void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
1483  HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
1484  HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
1485  Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
1486      nullptr : codegen_->GetLabelOf(true_successor);
1487  Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
1488      nullptr : codegen_->GetLabelOf(false_successor);
1489  GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
1490}
1491
1492void LocationsBuilderX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1493  LocationSummary* locations = new (GetGraph()->GetArena())
1494      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
1495  if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
1496    locations->SetInAt(0, Location::Any());
1497  }
1498}
1499
1500void InstructionCodeGeneratorX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1501  SlowPathCode* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathX86>(deoptimize);
1502  GenerateTestAndBranch<Label>(deoptimize,
1503                               /* condition_input_index */ 0,
1504                               slow_path->GetEntryLabel(),
1505                               /* false_target */ nullptr);
1506}
1507
1508static bool SelectCanUseCMOV(HSelect* select) {
1509  // There are no conditional move instructions for XMMs.
1510  if (Primitive::IsFloatingPointType(select->GetType())) {
1511    return false;
1512  }
1513
1514  // A FP condition doesn't generate the single CC that we need.
1515  // In 32 bit mode, a long condition doesn't generate a single CC either.
1516  HInstruction* condition = select->GetCondition();
1517  if (condition->IsCondition()) {
1518    Primitive::Type compare_type = condition->InputAt(0)->GetType();
1519    if (compare_type == Primitive::kPrimLong ||
1520        Primitive::IsFloatingPointType(compare_type)) {
1521      return false;
1522    }
1523  }
1524
1525  // We can generate a CMOV for this Select.
1526  return true;
1527}
1528
1529void LocationsBuilderX86::VisitSelect(HSelect* select) {
1530  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select);
1531  if (Primitive::IsFloatingPointType(select->GetType())) {
1532    locations->SetInAt(0, Location::RequiresFpuRegister());
1533    locations->SetInAt(1, Location::Any());
1534  } else {
1535    locations->SetInAt(0, Location::RequiresRegister());
1536    if (SelectCanUseCMOV(select)) {
1537      if (select->InputAt(1)->IsConstant()) {
1538        // Cmov can't handle a constant value.
1539        locations->SetInAt(1, Location::RequiresRegister());
1540      } else {
1541        locations->SetInAt(1, Location::Any());
1542      }
1543    } else {
1544      locations->SetInAt(1, Location::Any());
1545    }
1546  }
1547  if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) {
1548    locations->SetInAt(2, Location::RequiresRegister());
1549  }
1550  locations->SetOut(Location::SameAsFirstInput());
1551}
1552
1553void InstructionCodeGeneratorX86::GenerateIntCompare(Location lhs, Location rhs) {
1554  Register lhs_reg = lhs.AsRegister<Register>();
1555  if (rhs.IsConstant()) {
1556    int32_t value = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1557    codegen_->Compare32BitValue(lhs_reg, value);
1558  } else if (rhs.IsStackSlot()) {
1559    __ cmpl(lhs_reg, Address(ESP, rhs.GetStackIndex()));
1560  } else {
1561    __ cmpl(lhs_reg, rhs.AsRegister<Register>());
1562  }
1563}
1564
1565void InstructionCodeGeneratorX86::VisitSelect(HSelect* select) {
1566  LocationSummary* locations = select->GetLocations();
1567  DCHECK(locations->InAt(0).Equals(locations->Out()));
1568  if (SelectCanUseCMOV(select)) {
1569    // If both the condition and the source types are integer, we can generate
1570    // a CMOV to implement Select.
1571
1572    HInstruction* select_condition = select->GetCondition();
1573    Condition cond = kNotEqual;
1574
1575    // Figure out how to test the 'condition'.
1576    if (select_condition->IsCondition()) {
1577      HCondition* condition = select_condition->AsCondition();
1578      if (!condition->IsEmittedAtUseSite()) {
1579        // This was a previously materialized condition.
1580        // Can we use the existing condition code?
1581        if (AreEflagsSetFrom(condition, select)) {
1582          // Materialization was the previous instruction. Condition codes are right.
1583          cond = X86Condition(condition->GetCondition());
1584        } else {
1585          // No, we have to recreate the condition code.
1586          Register cond_reg = locations->InAt(2).AsRegister<Register>();
1587          __ testl(cond_reg, cond_reg);
1588        }
1589      } else {
1590        // We can't handle FP or long here.
1591        DCHECK_NE(condition->InputAt(0)->GetType(), Primitive::kPrimLong);
1592        DCHECK(!Primitive::IsFloatingPointType(condition->InputAt(0)->GetType()));
1593        LocationSummary* cond_locations = condition->GetLocations();
1594        GenerateIntCompare(cond_locations->InAt(0), cond_locations->InAt(1));
1595        cond = X86Condition(condition->GetCondition());
1596      }
1597    } else {
1598      // Must be a boolean condition, which needs to be compared to 0.
1599      Register cond_reg = locations->InAt(2).AsRegister<Register>();
1600      __ testl(cond_reg, cond_reg);
1601    }
1602
1603    // If the condition is true, overwrite the output, which already contains false.
1604    Location false_loc = locations->InAt(0);
1605    Location true_loc = locations->InAt(1);
1606    if (select->GetType() == Primitive::kPrimLong) {
1607      // 64 bit conditional move.
1608      Register false_high = false_loc.AsRegisterPairHigh<Register>();
1609      Register false_low = false_loc.AsRegisterPairLow<Register>();
1610      if (true_loc.IsRegisterPair()) {
1611        __ cmovl(cond, false_high, true_loc.AsRegisterPairHigh<Register>());
1612        __ cmovl(cond, false_low, true_loc.AsRegisterPairLow<Register>());
1613      } else {
1614        __ cmovl(cond, false_high, Address(ESP, true_loc.GetHighStackIndex(kX86WordSize)));
1615        __ cmovl(cond, false_low, Address(ESP, true_loc.GetStackIndex()));
1616      }
1617    } else {
1618      // 32 bit conditional move.
1619      Register false_reg = false_loc.AsRegister<Register>();
1620      if (true_loc.IsRegister()) {
1621        __ cmovl(cond, false_reg, true_loc.AsRegister<Register>());
1622      } else {
1623        __ cmovl(cond, false_reg, Address(ESP, true_loc.GetStackIndex()));
1624      }
1625    }
1626  } else {
1627    NearLabel false_target;
1628    GenerateTestAndBranch<NearLabel>(
1629        select, /* condition_input_index */ 2, /* true_target */ nullptr, &false_target);
1630    codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType());
1631    __ Bind(&false_target);
1632  }
1633}
1634
1635void LocationsBuilderX86::VisitNativeDebugInfo(HNativeDebugInfo* info) {
1636  new (GetGraph()->GetArena()) LocationSummary(info);
1637}
1638
1639void InstructionCodeGeneratorX86::VisitNativeDebugInfo(HNativeDebugInfo* info) {
1640  codegen_->MaybeRecordNativeDebugInfo(info, info->GetDexPc());
1641}
1642
1643void CodeGeneratorX86::GenerateNop() {
1644  __ nop();
1645}
1646
1647void LocationsBuilderX86::VisitLocal(HLocal* local) {
1648  local->SetLocations(nullptr);
1649}
1650
1651void InstructionCodeGeneratorX86::VisitLocal(HLocal* local) {
1652  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
1653}
1654
1655void LocationsBuilderX86::VisitLoadLocal(HLoadLocal* local) {
1656  local->SetLocations(nullptr);
1657}
1658
1659void InstructionCodeGeneratorX86::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_UNUSED) {
1660  // Nothing to do, this is driven by the code generator.
1661}
1662
1663void LocationsBuilderX86::VisitStoreLocal(HStoreLocal* store) {
1664  LocationSummary* locations =
1665      new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
1666  switch (store->InputAt(1)->GetType()) {
1667    case Primitive::kPrimBoolean:
1668    case Primitive::kPrimByte:
1669    case Primitive::kPrimChar:
1670    case Primitive::kPrimShort:
1671    case Primitive::kPrimInt:
1672    case Primitive::kPrimNot:
1673    case Primitive::kPrimFloat:
1674      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
1675      break;
1676
1677    case Primitive::kPrimLong:
1678    case Primitive::kPrimDouble:
1679      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
1680      break;
1681
1682    default:
1683      LOG(FATAL) << "Unknown local type " << store->InputAt(1)->GetType();
1684  }
1685}
1686
1687void InstructionCodeGeneratorX86::VisitStoreLocal(HStoreLocal* store ATTRIBUTE_UNUSED) {
1688}
1689
1690void LocationsBuilderX86::HandleCondition(HCondition* cond) {
1691  LocationSummary* locations =
1692      new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
1693  // Handle the long/FP comparisons made in instruction simplification.
1694  switch (cond->InputAt(0)->GetType()) {
1695    case Primitive::kPrimLong: {
1696      locations->SetInAt(0, Location::RequiresRegister());
1697      locations->SetInAt(1, Location::Any());
1698      if (!cond->IsEmittedAtUseSite()) {
1699        locations->SetOut(Location::RequiresRegister());
1700      }
1701      break;
1702    }
1703    case Primitive::kPrimFloat:
1704    case Primitive::kPrimDouble: {
1705      locations->SetInAt(0, Location::RequiresFpuRegister());
1706      if (cond->InputAt(1)->IsX86LoadFromConstantTable()) {
1707        DCHECK(cond->InputAt(1)->IsEmittedAtUseSite());
1708      } else if (cond->InputAt(1)->IsConstant()) {
1709        locations->SetInAt(1, Location::RequiresFpuRegister());
1710      } else {
1711        locations->SetInAt(1, Location::Any());
1712      }
1713      if (!cond->IsEmittedAtUseSite()) {
1714        locations->SetOut(Location::RequiresRegister());
1715      }
1716      break;
1717    }
1718    default:
1719      locations->SetInAt(0, Location::RequiresRegister());
1720      locations->SetInAt(1, Location::Any());
1721      if (!cond->IsEmittedAtUseSite()) {
1722        // We need a byte register.
1723        locations->SetOut(Location::RegisterLocation(ECX));
1724      }
1725      break;
1726  }
1727}
1728
1729void InstructionCodeGeneratorX86::HandleCondition(HCondition* cond) {
1730  if (cond->IsEmittedAtUseSite()) {
1731    return;
1732  }
1733
1734  LocationSummary* locations = cond->GetLocations();
1735  Location lhs = locations->InAt(0);
1736  Location rhs = locations->InAt(1);
1737  Register reg = locations->Out().AsRegister<Register>();
1738  NearLabel true_label, false_label;
1739
1740  switch (cond->InputAt(0)->GetType()) {
1741    default: {
1742      // Integer case.
1743
1744      // Clear output register: setb only sets the low byte.
1745      __ xorl(reg, reg);
1746      GenerateIntCompare(lhs, rhs);
1747      __ setb(X86Condition(cond->GetCondition()), reg);
1748      return;
1749    }
1750    case Primitive::kPrimLong:
1751      GenerateLongComparesAndJumps(cond, &true_label, &false_label);
1752      break;
1753    case Primitive::kPrimFloat:
1754      GenerateFPCompare(lhs, rhs, cond, false);
1755      GenerateFPJumps(cond, &true_label, &false_label);
1756      break;
1757    case Primitive::kPrimDouble:
1758      GenerateFPCompare(lhs, rhs, cond, true);
1759      GenerateFPJumps(cond, &true_label, &false_label);
1760      break;
1761  }
1762
1763  // Convert the jumps into the result.
1764  NearLabel done_label;
1765
1766  // False case: result = 0.
1767  __ Bind(&false_label);
1768  __ xorl(reg, reg);
1769  __ jmp(&done_label);
1770
1771  // True case: result = 1.
1772  __ Bind(&true_label);
1773  __ movl(reg, Immediate(1));
1774  __ Bind(&done_label);
1775}
1776
1777void LocationsBuilderX86::VisitEqual(HEqual* comp) {
1778  HandleCondition(comp);
1779}
1780
1781void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
1782  HandleCondition(comp);
1783}
1784
1785void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
1786  HandleCondition(comp);
1787}
1788
1789void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
1790  HandleCondition(comp);
1791}
1792
1793void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
1794  HandleCondition(comp);
1795}
1796
1797void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
1798  HandleCondition(comp);
1799}
1800
1801void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1802  HandleCondition(comp);
1803}
1804
1805void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1806  HandleCondition(comp);
1807}
1808
1809void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
1810  HandleCondition(comp);
1811}
1812
1813void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
1814  HandleCondition(comp);
1815}
1816
1817void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1818  HandleCondition(comp);
1819}
1820
1821void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1822  HandleCondition(comp);
1823}
1824
1825void LocationsBuilderX86::VisitBelow(HBelow* comp) {
1826  HandleCondition(comp);
1827}
1828
1829void InstructionCodeGeneratorX86::VisitBelow(HBelow* comp) {
1830  HandleCondition(comp);
1831}
1832
1833void LocationsBuilderX86::VisitBelowOrEqual(HBelowOrEqual* comp) {
1834  HandleCondition(comp);
1835}
1836
1837void InstructionCodeGeneratorX86::VisitBelowOrEqual(HBelowOrEqual* comp) {
1838  HandleCondition(comp);
1839}
1840
1841void LocationsBuilderX86::VisitAbove(HAbove* comp) {
1842  HandleCondition(comp);
1843}
1844
1845void InstructionCodeGeneratorX86::VisitAbove(HAbove* comp) {
1846  HandleCondition(comp);
1847}
1848
1849void LocationsBuilderX86::VisitAboveOrEqual(HAboveOrEqual* comp) {
1850  HandleCondition(comp);
1851}
1852
1853void InstructionCodeGeneratorX86::VisitAboveOrEqual(HAboveOrEqual* comp) {
1854  HandleCondition(comp);
1855}
1856
1857void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1858  LocationSummary* locations =
1859      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1860  locations->SetOut(Location::ConstantLocation(constant));
1861}
1862
1863void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) {
1864  // Will be generated at use site.
1865}
1866
1867void LocationsBuilderX86::VisitNullConstant(HNullConstant* constant) {
1868  LocationSummary* locations =
1869      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1870  locations->SetOut(Location::ConstantLocation(constant));
1871}
1872
1873void InstructionCodeGeneratorX86::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) {
1874  // Will be generated at use site.
1875}
1876
1877void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1878  LocationSummary* locations =
1879      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1880  locations->SetOut(Location::ConstantLocation(constant));
1881}
1882
1883void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) {
1884  // Will be generated at use site.
1885}
1886
1887void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1888  LocationSummary* locations =
1889      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1890  locations->SetOut(Location::ConstantLocation(constant));
1891}
1892
1893void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) {
1894  // Will be generated at use site.
1895}
1896
1897void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1898  LocationSummary* locations =
1899      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1900  locations->SetOut(Location::ConstantLocation(constant));
1901}
1902
1903void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) {
1904  // Will be generated at use site.
1905}
1906
1907void LocationsBuilderX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1908  memory_barrier->SetLocations(nullptr);
1909}
1910
1911void InstructionCodeGeneratorX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1912  codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1913}
1914
1915void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1916  ret->SetLocations(nullptr);
1917}
1918
1919void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) {
1920  codegen_->GenerateFrameExit();
1921}
1922
1923void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1924  LocationSummary* locations =
1925      new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1926  switch (ret->InputAt(0)->GetType()) {
1927    case Primitive::kPrimBoolean:
1928    case Primitive::kPrimByte:
1929    case Primitive::kPrimChar:
1930    case Primitive::kPrimShort:
1931    case Primitive::kPrimInt:
1932    case Primitive::kPrimNot:
1933      locations->SetInAt(0, Location::RegisterLocation(EAX));
1934      break;
1935
1936    case Primitive::kPrimLong:
1937      locations->SetInAt(
1938          0, Location::RegisterPairLocation(EAX, EDX));
1939      break;
1940
1941    case Primitive::kPrimFloat:
1942    case Primitive::kPrimDouble:
1943      locations->SetInAt(
1944          0, Location::FpuRegisterLocation(XMM0));
1945      break;
1946
1947    default:
1948      LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1949  }
1950}
1951
1952void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1953  if (kIsDebugBuild) {
1954    switch (ret->InputAt(0)->GetType()) {
1955      case Primitive::kPrimBoolean:
1956      case Primitive::kPrimByte:
1957      case Primitive::kPrimChar:
1958      case Primitive::kPrimShort:
1959      case Primitive::kPrimInt:
1960      case Primitive::kPrimNot:
1961        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1962        break;
1963
1964      case Primitive::kPrimLong:
1965        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1966        DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1967        break;
1968
1969      case Primitive::kPrimFloat:
1970      case Primitive::kPrimDouble:
1971        DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1972        break;
1973
1974      default:
1975        LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1976    }
1977  }
1978  codegen_->GenerateFrameExit();
1979}
1980
1981void LocationsBuilderX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1982  // The trampoline uses the same calling convention as dex calling conventions,
1983  // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain
1984  // the method_idx.
1985  HandleInvoke(invoke);
1986}
1987
1988void InstructionCodeGeneratorX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1989  codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
1990}
1991
1992void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1993  // Explicit clinit checks triggered by static invokes must have been pruned by
1994  // art::PrepareForRegisterAllocation.
1995  DCHECK(!invoke->IsStaticWithExplicitClinitCheck());
1996
1997  IntrinsicLocationsBuilderX86 intrinsic(codegen_);
1998  if (intrinsic.TryDispatch(invoke)) {
1999    if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) {
2000      invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any());
2001    }
2002    return;
2003  }
2004
2005  HandleInvoke(invoke);
2006
2007  // For PC-relative dex cache the invoke has an extra input, the PC-relative address base.
2008  if (invoke->HasPcRelativeDexCache()) {
2009    invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister());
2010  }
2011}
2012
2013static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) {
2014  if (invoke->GetLocations()->Intrinsified()) {
2015    IntrinsicCodeGeneratorX86 intrinsic(codegen);
2016    intrinsic.Dispatch(invoke);
2017    return true;
2018  }
2019  return false;
2020}
2021
2022void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
2023  // Explicit clinit checks triggered by static invokes must have been pruned by
2024  // art::PrepareForRegisterAllocation.
2025  DCHECK(!invoke->IsStaticWithExplicitClinitCheck());
2026
2027  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
2028    return;
2029  }
2030
2031  LocationSummary* locations = invoke->GetLocations();
2032  codegen_->GenerateStaticOrDirectCall(
2033      invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
2034  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2035}
2036
2037void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
2038  IntrinsicLocationsBuilderX86 intrinsic(codegen_);
2039  if (intrinsic.TryDispatch(invoke)) {
2040    return;
2041  }
2042
2043  HandleInvoke(invoke);
2044}
2045
2046void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
2047  InvokeDexCallingConventionVisitorX86 calling_convention_visitor;
2048  CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
2049}
2050
2051void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
2052  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
2053    return;
2054  }
2055
2056  codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
2057  DCHECK(!codegen_->IsLeafMethod());
2058  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2059}
2060
2061void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
2062  // This call to HandleInvoke allocates a temporary (core) register
2063  // which is also used to transfer the hidden argument from FP to
2064  // core register.
2065  HandleInvoke(invoke);
2066  // Add the hidden argument.
2067  invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
2068}
2069
2070void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
2071  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
2072  LocationSummary* locations = invoke->GetLocations();
2073  Register temp = locations->GetTemp(0).AsRegister<Register>();
2074  XmmRegister hidden_reg = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
2075  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
2076      invoke->GetImtIndex() % mirror::Class::kImtSize, kX86PointerSize).Uint32Value();
2077  Location receiver = locations->InAt(0);
2078  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
2079
2080  // Set the hidden argument. This is safe to do this here, as XMM7
2081  // won't be modified thereafter, before the `call` instruction.
2082  DCHECK_EQ(XMM7, hidden_reg);
2083  __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
2084  __ movd(hidden_reg, temp);
2085
2086  if (receiver.IsStackSlot()) {
2087    __ movl(temp, Address(ESP, receiver.GetStackIndex()));
2088    // /* HeapReference<Class> */ temp = temp->klass_
2089    __ movl(temp, Address(temp, class_offset));
2090  } else {
2091    // /* HeapReference<Class> */ temp = receiver->klass_
2092    __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
2093  }
2094  codegen_->MaybeRecordImplicitNullCheck(invoke);
2095  // Instead of simply (possibly) unpoisoning `temp` here, we should
2096  // emit a read barrier for the previous class reference load.
2097  // However this is not required in practice, as this is an
2098  // intermediate/temporary reference and because the current
2099  // concurrent copying collector keeps the from-space memory
2100  // intact/accessible until the end of the marking phase (the
2101  // concurrent copying collector may not in the future).
2102  __ MaybeUnpoisonHeapReference(temp);
2103  // temp = temp->GetImtEntryAt(method_offset);
2104  __ movl(temp, Address(temp, method_offset));
2105  // call temp->GetEntryPoint();
2106  __ call(Address(temp,
2107                  ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
2108
2109  DCHECK(!codegen_->IsLeafMethod());
2110  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2111}
2112
2113void LocationsBuilderX86::VisitNeg(HNeg* neg) {
2114  LocationSummary* locations =
2115      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
2116  switch (neg->GetResultType()) {
2117    case Primitive::kPrimInt:
2118    case Primitive::kPrimLong:
2119      locations->SetInAt(0, Location::RequiresRegister());
2120      locations->SetOut(Location::SameAsFirstInput());
2121      break;
2122
2123    case Primitive::kPrimFloat:
2124      locations->SetInAt(0, Location::RequiresFpuRegister());
2125      locations->SetOut(Location::SameAsFirstInput());
2126      locations->AddTemp(Location::RequiresRegister());
2127      locations->AddTemp(Location::RequiresFpuRegister());
2128      break;
2129
2130    case Primitive::kPrimDouble:
2131      locations->SetInAt(0, Location::RequiresFpuRegister());
2132      locations->SetOut(Location::SameAsFirstInput());
2133      locations->AddTemp(Location::RequiresFpuRegister());
2134      break;
2135
2136    default:
2137      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
2138  }
2139}
2140
2141void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
2142  LocationSummary* locations = neg->GetLocations();
2143  Location out = locations->Out();
2144  Location in = locations->InAt(0);
2145  switch (neg->GetResultType()) {
2146    case Primitive::kPrimInt:
2147      DCHECK(in.IsRegister());
2148      DCHECK(in.Equals(out));
2149      __ negl(out.AsRegister<Register>());
2150      break;
2151
2152    case Primitive::kPrimLong:
2153      DCHECK(in.IsRegisterPair());
2154      DCHECK(in.Equals(out));
2155      __ negl(out.AsRegisterPairLow<Register>());
2156      // Negation is similar to subtraction from zero.  The least
2157      // significant byte triggers a borrow when it is different from
2158      // zero; to take it into account, add 1 to the most significant
2159      // byte if the carry flag (CF) is set to 1 after the first NEGL
2160      // operation.
2161      __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
2162      __ negl(out.AsRegisterPairHigh<Register>());
2163      break;
2164
2165    case Primitive::kPrimFloat: {
2166      DCHECK(in.Equals(out));
2167      Register constant = locations->GetTemp(0).AsRegister<Register>();
2168      XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
2169      // Implement float negation with an exclusive or with value
2170      // 0x80000000 (mask for bit 31, representing the sign of a
2171      // single-precision floating-point number).
2172      __ movl(constant, Immediate(INT32_C(0x80000000)));
2173      __ movd(mask, constant);
2174      __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
2175      break;
2176    }
2177
2178    case Primitive::kPrimDouble: {
2179      DCHECK(in.Equals(out));
2180      XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2181      // Implement double negation with an exclusive or with value
2182      // 0x8000000000000000 (mask for bit 63, representing the sign of
2183      // a double-precision floating-point number).
2184      __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
2185      __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
2186      break;
2187    }
2188
2189    default:
2190      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
2191  }
2192}
2193
2194void LocationsBuilderX86::VisitX86FPNeg(HX86FPNeg* neg) {
2195  LocationSummary* locations =
2196      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
2197  DCHECK(Primitive::IsFloatingPointType(neg->GetType()));
2198  locations->SetInAt(0, Location::RequiresFpuRegister());
2199  locations->SetInAt(1, Location::RequiresRegister());
2200  locations->SetOut(Location::SameAsFirstInput());
2201  locations->AddTemp(Location::RequiresFpuRegister());
2202}
2203
2204void InstructionCodeGeneratorX86::VisitX86FPNeg(HX86FPNeg* neg) {
2205  LocationSummary* locations = neg->GetLocations();
2206  Location out = locations->Out();
2207  DCHECK(locations->InAt(0).Equals(out));
2208
2209  Register constant_area = locations->InAt(1).AsRegister<Register>();
2210  XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2211  if (neg->GetType() == Primitive::kPrimFloat) {
2212    __ movss(mask, codegen_->LiteralInt32Address(INT32_C(0x80000000), constant_area));
2213    __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
2214  } else {
2215     __ movsd(mask, codegen_->LiteralInt64Address(INT64_C(0x8000000000000000), constant_area));
2216     __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
2217  }
2218}
2219
2220void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
2221  Primitive::Type result_type = conversion->GetResultType();
2222  Primitive::Type input_type = conversion->GetInputType();
2223  DCHECK_NE(result_type, input_type);
2224
2225  // The float-to-long and double-to-long type conversions rely on a
2226  // call to the runtime.
2227  LocationSummary::CallKind call_kind =
2228      ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
2229       && result_type == Primitive::kPrimLong)
2230      ? LocationSummary::kCall
2231      : LocationSummary::kNoCall;
2232  LocationSummary* locations =
2233      new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
2234
2235  // The Java language does not allow treating boolean as an integral type but
2236  // our bit representation makes it safe.
2237
2238  switch (result_type) {
2239    case Primitive::kPrimByte:
2240      switch (input_type) {
2241        case Primitive::kPrimLong: {
2242          // Type conversion from long to byte is a result of code transformations.
2243          HInstruction* input = conversion->InputAt(0);
2244          Location input_location = input->IsConstant()
2245              ? Location::ConstantLocation(input->AsConstant())
2246              : Location::RegisterPairLocation(EAX, EDX);
2247          locations->SetInAt(0, input_location);
2248          // Make the output overlap to please the register allocator. This greatly simplifies
2249          // the validation of the linear scan implementation
2250          locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
2251          break;
2252        }
2253        case Primitive::kPrimBoolean:
2254          // Boolean input is a result of code transformations.
2255        case Primitive::kPrimShort:
2256        case Primitive::kPrimInt:
2257        case Primitive::kPrimChar:
2258          // Processing a Dex `int-to-byte' instruction.
2259          locations->SetInAt(0, Location::ByteRegisterOrConstant(ECX, conversion->InputAt(0)));
2260          // Make the output overlap to please the register allocator. This greatly simplifies
2261          // the validation of the linear scan implementation
2262          locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
2263          break;
2264
2265        default:
2266          LOG(FATAL) << "Unexpected type conversion from " << input_type
2267                     << " to " << result_type;
2268      }
2269      break;
2270
2271    case Primitive::kPrimShort:
2272      switch (input_type) {
2273        case Primitive::kPrimLong:
2274          // Type conversion from long to short is a result of code transformations.
2275        case Primitive::kPrimBoolean:
2276          // Boolean input is a result of code transformations.
2277        case Primitive::kPrimByte:
2278        case Primitive::kPrimInt:
2279        case Primitive::kPrimChar:
2280          // Processing a Dex `int-to-short' instruction.
2281          locations->SetInAt(0, Location::Any());
2282          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2283          break;
2284
2285        default:
2286          LOG(FATAL) << "Unexpected type conversion from " << input_type
2287                     << " to " << result_type;
2288      }
2289      break;
2290
2291    case Primitive::kPrimInt:
2292      switch (input_type) {
2293        case Primitive::kPrimLong:
2294          // Processing a Dex `long-to-int' instruction.
2295          locations->SetInAt(0, Location::Any());
2296          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2297          break;
2298
2299        case Primitive::kPrimFloat:
2300          // Processing a Dex `float-to-int' instruction.
2301          locations->SetInAt(0, Location::RequiresFpuRegister());
2302          locations->SetOut(Location::RequiresRegister());
2303          locations->AddTemp(Location::RequiresFpuRegister());
2304          break;
2305
2306        case Primitive::kPrimDouble:
2307          // Processing a Dex `double-to-int' instruction.
2308          locations->SetInAt(0, Location::RequiresFpuRegister());
2309          locations->SetOut(Location::RequiresRegister());
2310          locations->AddTemp(Location::RequiresFpuRegister());
2311          break;
2312
2313        default:
2314          LOG(FATAL) << "Unexpected type conversion from " << input_type
2315                     << " to " << result_type;
2316      }
2317      break;
2318
2319    case Primitive::kPrimLong:
2320      switch (input_type) {
2321        case Primitive::kPrimBoolean:
2322          // Boolean input is a result of code transformations.
2323        case Primitive::kPrimByte:
2324        case Primitive::kPrimShort:
2325        case Primitive::kPrimInt:
2326        case Primitive::kPrimChar:
2327          // Processing a Dex `int-to-long' instruction.
2328          locations->SetInAt(0, Location::RegisterLocation(EAX));
2329          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2330          break;
2331
2332        case Primitive::kPrimFloat:
2333        case Primitive::kPrimDouble: {
2334          // Processing a Dex `float-to-long' or 'double-to-long' instruction.
2335          InvokeRuntimeCallingConvention calling_convention;
2336          XmmRegister parameter = calling_convention.GetFpuRegisterAt(0);
2337          locations->SetInAt(0, Location::FpuRegisterLocation(parameter));
2338
2339          // The runtime helper puts the result in EAX, EDX.
2340          locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2341        }
2342        break;
2343
2344        default:
2345          LOG(FATAL) << "Unexpected type conversion from " << input_type
2346                     << " to " << result_type;
2347      }
2348      break;
2349
2350    case Primitive::kPrimChar:
2351      switch (input_type) {
2352        case Primitive::kPrimLong:
2353          // Type conversion from long to char is a result of code transformations.
2354        case Primitive::kPrimBoolean:
2355          // Boolean input is a result of code transformations.
2356        case Primitive::kPrimByte:
2357        case Primitive::kPrimShort:
2358        case Primitive::kPrimInt:
2359          // Processing a Dex `int-to-char' instruction.
2360          locations->SetInAt(0, Location::Any());
2361          locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2362          break;
2363
2364        default:
2365          LOG(FATAL) << "Unexpected type conversion from " << input_type
2366                     << " to " << result_type;
2367      }
2368      break;
2369
2370    case Primitive::kPrimFloat:
2371      switch (input_type) {
2372        case Primitive::kPrimBoolean:
2373          // Boolean input is a result of code transformations.
2374        case Primitive::kPrimByte:
2375        case Primitive::kPrimShort:
2376        case Primitive::kPrimInt:
2377        case Primitive::kPrimChar:
2378          // Processing a Dex `int-to-float' instruction.
2379          locations->SetInAt(0, Location::RequiresRegister());
2380          locations->SetOut(Location::RequiresFpuRegister());
2381          break;
2382
2383        case Primitive::kPrimLong:
2384          // Processing a Dex `long-to-float' instruction.
2385          locations->SetInAt(0, Location::Any());
2386          locations->SetOut(Location::Any());
2387          break;
2388
2389        case Primitive::kPrimDouble:
2390          // Processing a Dex `double-to-float' instruction.
2391          locations->SetInAt(0, Location::RequiresFpuRegister());
2392          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2393          break;
2394
2395        default:
2396          LOG(FATAL) << "Unexpected type conversion from " << input_type
2397                     << " to " << result_type;
2398      };
2399      break;
2400
2401    case Primitive::kPrimDouble:
2402      switch (input_type) {
2403        case Primitive::kPrimBoolean:
2404          // Boolean input is a result of code transformations.
2405        case Primitive::kPrimByte:
2406        case Primitive::kPrimShort:
2407        case Primitive::kPrimInt:
2408        case Primitive::kPrimChar:
2409          // Processing a Dex `int-to-double' instruction.
2410          locations->SetInAt(0, Location::RequiresRegister());
2411          locations->SetOut(Location::RequiresFpuRegister());
2412          break;
2413
2414        case Primitive::kPrimLong:
2415          // Processing a Dex `long-to-double' instruction.
2416          locations->SetInAt(0, Location::Any());
2417          locations->SetOut(Location::Any());
2418          break;
2419
2420        case Primitive::kPrimFloat:
2421          // Processing a Dex `float-to-double' instruction.
2422          locations->SetInAt(0, Location::RequiresFpuRegister());
2423          locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2424          break;
2425
2426        default:
2427          LOG(FATAL) << "Unexpected type conversion from " << input_type
2428                     << " to " << result_type;
2429      }
2430      break;
2431
2432    default:
2433      LOG(FATAL) << "Unexpected type conversion from " << input_type
2434                 << " to " << result_type;
2435  }
2436}
2437
2438void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
2439  LocationSummary* locations = conversion->GetLocations();
2440  Location out = locations->Out();
2441  Location in = locations->InAt(0);
2442  Primitive::Type result_type = conversion->GetResultType();
2443  Primitive::Type input_type = conversion->GetInputType();
2444  DCHECK_NE(result_type, input_type);
2445  switch (result_type) {
2446    case Primitive::kPrimByte:
2447      switch (input_type) {
2448        case Primitive::kPrimLong:
2449          // Type conversion from long to byte is a result of code transformations.
2450          if (in.IsRegisterPair()) {
2451            __ movsxb(out.AsRegister<Register>(), in.AsRegisterPairLow<ByteRegister>());
2452          } else {
2453            DCHECK(in.GetConstant()->IsLongConstant());
2454            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
2455            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
2456          }
2457          break;
2458        case Primitive::kPrimBoolean:
2459          // Boolean input is a result of code transformations.
2460        case Primitive::kPrimShort:
2461        case Primitive::kPrimInt:
2462        case Primitive::kPrimChar:
2463          // Processing a Dex `int-to-byte' instruction.
2464          if (in.IsRegister()) {
2465            __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
2466          } else {
2467            DCHECK(in.GetConstant()->IsIntConstant());
2468            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2469            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
2470          }
2471          break;
2472
2473        default:
2474          LOG(FATAL) << "Unexpected type conversion from " << input_type
2475                     << " to " << result_type;
2476      }
2477      break;
2478
2479    case Primitive::kPrimShort:
2480      switch (input_type) {
2481        case Primitive::kPrimLong:
2482          // Type conversion from long to short is a result of code transformations.
2483          if (in.IsRegisterPair()) {
2484            __ movsxw(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
2485          } else if (in.IsDoubleStackSlot()) {
2486            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2487          } else {
2488            DCHECK(in.GetConstant()->IsLongConstant());
2489            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
2490            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
2491          }
2492          break;
2493        case Primitive::kPrimBoolean:
2494          // Boolean input is a result of code transformations.
2495        case Primitive::kPrimByte:
2496        case Primitive::kPrimInt:
2497        case Primitive::kPrimChar:
2498          // Processing a Dex `int-to-short' instruction.
2499          if (in.IsRegister()) {
2500            __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2501          } else if (in.IsStackSlot()) {
2502            __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2503          } else {
2504            DCHECK(in.GetConstant()->IsIntConstant());
2505            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2506            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
2507          }
2508          break;
2509
2510        default:
2511          LOG(FATAL) << "Unexpected type conversion from " << input_type
2512                     << " to " << result_type;
2513      }
2514      break;
2515
2516    case Primitive::kPrimInt:
2517      switch (input_type) {
2518        case Primitive::kPrimLong:
2519          // Processing a Dex `long-to-int' instruction.
2520          if (in.IsRegisterPair()) {
2521            __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
2522          } else if (in.IsDoubleStackSlot()) {
2523            __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2524          } else {
2525            DCHECK(in.IsConstant());
2526            DCHECK(in.GetConstant()->IsLongConstant());
2527            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
2528            __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
2529          }
2530          break;
2531
2532        case Primitive::kPrimFloat: {
2533          // Processing a Dex `float-to-int' instruction.
2534          XmmRegister input = in.AsFpuRegister<XmmRegister>();
2535          Register output = out.AsRegister<Register>();
2536          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2537          NearLabel done, nan;
2538
2539          __ movl(output, Immediate(kPrimIntMax));
2540          // temp = int-to-float(output)
2541          __ cvtsi2ss(temp, output);
2542          // if input >= temp goto done
2543          __ comiss(input, temp);
2544          __ j(kAboveEqual, &done);
2545          // if input == NaN goto nan
2546          __ j(kUnordered, &nan);
2547          // output = float-to-int-truncate(input)
2548          __ cvttss2si(output, input);
2549          __ jmp(&done);
2550          __ Bind(&nan);
2551          //  output = 0
2552          __ xorl(output, output);
2553          __ Bind(&done);
2554          break;
2555        }
2556
2557        case Primitive::kPrimDouble: {
2558          // Processing a Dex `double-to-int' instruction.
2559          XmmRegister input = in.AsFpuRegister<XmmRegister>();
2560          Register output = out.AsRegister<Register>();
2561          XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2562          NearLabel done, nan;
2563
2564          __ movl(output, Immediate(kPrimIntMax));
2565          // temp = int-to-double(output)
2566          __ cvtsi2sd(temp, output);
2567          // if input >= temp goto done
2568          __ comisd(input, temp);
2569          __ j(kAboveEqual, &done);
2570          // if input == NaN goto nan
2571          __ j(kUnordered, &nan);
2572          // output = double-to-int-truncate(input)
2573          __ cvttsd2si(output, input);
2574          __ jmp(&done);
2575          __ Bind(&nan);
2576          //  output = 0
2577          __ xorl(output, output);
2578          __ Bind(&done);
2579          break;
2580        }
2581
2582        default:
2583          LOG(FATAL) << "Unexpected type conversion from " << input_type
2584                     << " to " << result_type;
2585      }
2586      break;
2587
2588    case Primitive::kPrimLong:
2589      switch (input_type) {
2590        case Primitive::kPrimBoolean:
2591          // Boolean input is a result of code transformations.
2592        case Primitive::kPrimByte:
2593        case Primitive::kPrimShort:
2594        case Primitive::kPrimInt:
2595        case Primitive::kPrimChar:
2596          // Processing a Dex `int-to-long' instruction.
2597          DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
2598          DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
2599          DCHECK_EQ(in.AsRegister<Register>(), EAX);
2600          __ cdq();
2601          break;
2602
2603        case Primitive::kPrimFloat:
2604          // Processing a Dex `float-to-long' instruction.
2605          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pF2l),
2606                                  conversion,
2607                                  conversion->GetDexPc(),
2608                                  nullptr);
2609          CheckEntrypointTypes<kQuickF2l, int64_t, float>();
2610          break;
2611
2612        case Primitive::kPrimDouble:
2613          // Processing a Dex `double-to-long' instruction.
2614          codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pD2l),
2615                                  conversion,
2616                                  conversion->GetDexPc(),
2617                                  nullptr);
2618          CheckEntrypointTypes<kQuickD2l, int64_t, double>();
2619          break;
2620
2621        default:
2622          LOG(FATAL) << "Unexpected type conversion from " << input_type
2623                     << " to " << result_type;
2624      }
2625      break;
2626
2627    case Primitive::kPrimChar:
2628      switch (input_type) {
2629        case Primitive::kPrimLong:
2630          // Type conversion from long to short is a result of code transformations.
2631          if (in.IsRegisterPair()) {
2632            __ movzxw(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
2633          } else if (in.IsDoubleStackSlot()) {
2634            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2635          } else {
2636            DCHECK(in.GetConstant()->IsLongConstant());
2637            int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
2638            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
2639          }
2640          break;
2641        case Primitive::kPrimBoolean:
2642          // Boolean input is a result of code transformations.
2643        case Primitive::kPrimByte:
2644        case Primitive::kPrimShort:
2645        case Primitive::kPrimInt:
2646          // Processing a Dex `Process a Dex `int-to-char'' instruction.
2647          if (in.IsRegister()) {
2648            __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2649          } else if (in.IsStackSlot()) {
2650            __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2651          } else {
2652            DCHECK(in.GetConstant()->IsIntConstant());
2653            int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2654            __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
2655          }
2656          break;
2657
2658        default:
2659          LOG(FATAL) << "Unexpected type conversion from " << input_type
2660                     << " to " << result_type;
2661      }
2662      break;
2663
2664    case Primitive::kPrimFloat:
2665      switch (input_type) {
2666        case Primitive::kPrimBoolean:
2667          // Boolean input is a result of code transformations.
2668        case Primitive::kPrimByte:
2669        case Primitive::kPrimShort:
2670        case Primitive::kPrimInt:
2671        case Primitive::kPrimChar:
2672          // Processing a Dex `int-to-float' instruction.
2673          __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2674          break;
2675
2676        case Primitive::kPrimLong: {
2677          // Processing a Dex `long-to-float' instruction.
2678          size_t adjustment = 0;
2679
2680          // Create stack space for the call to
2681          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstps below.
2682          // TODO: enhance register allocator to ask for stack temporaries.
2683          if (!in.IsDoubleStackSlot() || !out.IsStackSlot()) {
2684            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2685            __ subl(ESP, Immediate(adjustment));
2686          }
2687
2688          // Load the value to the FP stack, using temporaries if needed.
2689          PushOntoFPStack(in, 0, adjustment, false, true);
2690
2691          if (out.IsStackSlot()) {
2692            __ fstps(Address(ESP, out.GetStackIndex() + adjustment));
2693          } else {
2694            __ fstps(Address(ESP, 0));
2695            Location stack_temp = Location::StackSlot(0);
2696            codegen_->Move32(out, stack_temp);
2697          }
2698
2699          // Remove the temporary stack space we allocated.
2700          if (adjustment != 0) {
2701            __ addl(ESP, Immediate(adjustment));
2702          }
2703          break;
2704        }
2705
2706        case Primitive::kPrimDouble:
2707          // Processing a Dex `double-to-float' instruction.
2708          __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2709          break;
2710
2711        default:
2712          LOG(FATAL) << "Unexpected type conversion from " << input_type
2713                     << " to " << result_type;
2714      };
2715      break;
2716
2717    case Primitive::kPrimDouble:
2718      switch (input_type) {
2719        case Primitive::kPrimBoolean:
2720          // Boolean input is a result of code transformations.
2721        case Primitive::kPrimByte:
2722        case Primitive::kPrimShort:
2723        case Primitive::kPrimInt:
2724        case Primitive::kPrimChar:
2725          // Processing a Dex `int-to-double' instruction.
2726          __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2727          break;
2728
2729        case Primitive::kPrimLong: {
2730          // Processing a Dex `long-to-double' instruction.
2731          size_t adjustment = 0;
2732
2733          // Create stack space for the call to
2734          // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstpl below.
2735          // TODO: enhance register allocator to ask for stack temporaries.
2736          if (!in.IsDoubleStackSlot() || !out.IsDoubleStackSlot()) {
2737            adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2738            __ subl(ESP, Immediate(adjustment));
2739          }
2740
2741          // Load the value to the FP stack, using temporaries if needed.
2742          PushOntoFPStack(in, 0, adjustment, false, true);
2743
2744          if (out.IsDoubleStackSlot()) {
2745            __ fstpl(Address(ESP, out.GetStackIndex() + adjustment));
2746          } else {
2747            __ fstpl(Address(ESP, 0));
2748            Location stack_temp = Location::DoubleStackSlot(0);
2749            codegen_->Move64(out, stack_temp);
2750          }
2751
2752          // Remove the temporary stack space we allocated.
2753          if (adjustment != 0) {
2754            __ addl(ESP, Immediate(adjustment));
2755          }
2756          break;
2757        }
2758
2759        case Primitive::kPrimFloat:
2760          // Processing a Dex `float-to-double' instruction.
2761          __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2762          break;
2763
2764        default:
2765          LOG(FATAL) << "Unexpected type conversion from " << input_type
2766                     << " to " << result_type;
2767      };
2768      break;
2769
2770    default:
2771      LOG(FATAL) << "Unexpected type conversion from " << input_type
2772                 << " to " << result_type;
2773  }
2774}
2775
2776void LocationsBuilderX86::VisitAdd(HAdd* add) {
2777  LocationSummary* locations =
2778      new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
2779  switch (add->GetResultType()) {
2780    case Primitive::kPrimInt: {
2781      locations->SetInAt(0, Location::RequiresRegister());
2782      locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
2783      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2784      break;
2785    }
2786
2787    case Primitive::kPrimLong: {
2788      locations->SetInAt(0, Location::RequiresRegister());
2789      locations->SetInAt(1, Location::Any());
2790      locations->SetOut(Location::SameAsFirstInput());
2791      break;
2792    }
2793
2794    case Primitive::kPrimFloat:
2795    case Primitive::kPrimDouble: {
2796      locations->SetInAt(0, Location::RequiresFpuRegister());
2797      if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2798        DCHECK(add->InputAt(1)->IsEmittedAtUseSite());
2799      } else if (add->InputAt(1)->IsConstant()) {
2800        locations->SetInAt(1, Location::RequiresFpuRegister());
2801      } else {
2802        locations->SetInAt(1, Location::Any());
2803      }
2804      locations->SetOut(Location::SameAsFirstInput());
2805      break;
2806    }
2807
2808    default:
2809      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2810      break;
2811  }
2812}
2813
2814void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
2815  LocationSummary* locations = add->GetLocations();
2816  Location first = locations->InAt(0);
2817  Location second = locations->InAt(1);
2818  Location out = locations->Out();
2819
2820  switch (add->GetResultType()) {
2821    case Primitive::kPrimInt: {
2822      if (second.IsRegister()) {
2823        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2824          __ addl(out.AsRegister<Register>(), second.AsRegister<Register>());
2825        } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
2826          __ addl(out.AsRegister<Register>(), first.AsRegister<Register>());
2827        } else {
2828          __ leal(out.AsRegister<Register>(), Address(
2829              first.AsRegister<Register>(), second.AsRegister<Register>(), TIMES_1, 0));
2830          }
2831      } else if (second.IsConstant()) {
2832        int32_t value = second.GetConstant()->AsIntConstant()->GetValue();
2833        if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2834          __ addl(out.AsRegister<Register>(), Immediate(value));
2835        } else {
2836          __ leal(out.AsRegister<Register>(), Address(first.AsRegister<Register>(), value));
2837        }
2838      } else {
2839        DCHECK(first.Equals(locations->Out()));
2840        __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2841      }
2842      break;
2843    }
2844
2845    case Primitive::kPrimLong: {
2846      if (second.IsRegisterPair()) {
2847        __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2848        __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2849      } else if (second.IsDoubleStackSlot()) {
2850        __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2851        __ adcl(first.AsRegisterPairHigh<Register>(),
2852                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2853      } else {
2854        DCHECK(second.IsConstant()) << second;
2855        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2856        __ addl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2857        __ adcl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2858      }
2859      break;
2860    }
2861
2862    case Primitive::kPrimFloat: {
2863      if (second.IsFpuRegister()) {
2864        __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2865      } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2866        HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2867        DCHECK(const_area->IsEmittedAtUseSite());
2868        __ addss(first.AsFpuRegister<XmmRegister>(),
2869                 codegen_->LiteralFloatAddress(
2870                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2871                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2872      } else {
2873        DCHECK(second.IsStackSlot());
2874        __ addss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2875      }
2876      break;
2877    }
2878
2879    case Primitive::kPrimDouble: {
2880      if (second.IsFpuRegister()) {
2881        __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2882      } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2883        HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2884        DCHECK(const_area->IsEmittedAtUseSite());
2885        __ addsd(first.AsFpuRegister<XmmRegister>(),
2886                 codegen_->LiteralDoubleAddress(
2887                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2888                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2889      } else {
2890        DCHECK(second.IsDoubleStackSlot());
2891        __ addsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2892      }
2893      break;
2894    }
2895
2896    default:
2897      LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2898  }
2899}
2900
2901void LocationsBuilderX86::VisitSub(HSub* sub) {
2902  LocationSummary* locations =
2903      new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2904  switch (sub->GetResultType()) {
2905    case Primitive::kPrimInt:
2906    case Primitive::kPrimLong: {
2907      locations->SetInAt(0, Location::RequiresRegister());
2908      locations->SetInAt(1, Location::Any());
2909      locations->SetOut(Location::SameAsFirstInput());
2910      break;
2911    }
2912    case Primitive::kPrimFloat:
2913    case Primitive::kPrimDouble: {
2914      locations->SetInAt(0, Location::RequiresFpuRegister());
2915      if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2916        DCHECK(sub->InputAt(1)->IsEmittedAtUseSite());
2917      } else if (sub->InputAt(1)->IsConstant()) {
2918        locations->SetInAt(1, Location::RequiresFpuRegister());
2919      } else {
2920        locations->SetInAt(1, Location::Any());
2921      }
2922      locations->SetOut(Location::SameAsFirstInput());
2923      break;
2924    }
2925
2926    default:
2927      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2928  }
2929}
2930
2931void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
2932  LocationSummary* locations = sub->GetLocations();
2933  Location first = locations->InAt(0);
2934  Location second = locations->InAt(1);
2935  DCHECK(first.Equals(locations->Out()));
2936  switch (sub->GetResultType()) {
2937    case Primitive::kPrimInt: {
2938      if (second.IsRegister()) {
2939        __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
2940      } else if (second.IsConstant()) {
2941        __ subl(first.AsRegister<Register>(),
2942                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2943      } else {
2944        __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2945      }
2946      break;
2947    }
2948
2949    case Primitive::kPrimLong: {
2950      if (second.IsRegisterPair()) {
2951        __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2952        __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2953      } else if (second.IsDoubleStackSlot()) {
2954        __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2955        __ sbbl(first.AsRegisterPairHigh<Register>(),
2956                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2957      } else {
2958        DCHECK(second.IsConstant()) << second;
2959        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2960        __ subl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2961        __ sbbl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2962      }
2963      break;
2964    }
2965
2966    case Primitive::kPrimFloat: {
2967      if (second.IsFpuRegister()) {
2968        __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2969      } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2970        HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2971        DCHECK(const_area->IsEmittedAtUseSite());
2972        __ subss(first.AsFpuRegister<XmmRegister>(),
2973                 codegen_->LiteralFloatAddress(
2974                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
2975                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2976      } else {
2977        DCHECK(second.IsStackSlot());
2978        __ subss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2979      }
2980      break;
2981    }
2982
2983    case Primitive::kPrimDouble: {
2984      if (second.IsFpuRegister()) {
2985        __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2986      } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2987        HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2988        DCHECK(const_area->IsEmittedAtUseSite());
2989        __ subsd(first.AsFpuRegister<XmmRegister>(),
2990                 codegen_->LiteralDoubleAddress(
2991                     const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2992                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2993      } else {
2994        DCHECK(second.IsDoubleStackSlot());
2995        __ subsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2996      }
2997      break;
2998    }
2999
3000    default:
3001      LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
3002  }
3003}
3004
3005void LocationsBuilderX86::VisitMul(HMul* mul) {
3006  LocationSummary* locations =
3007      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
3008  switch (mul->GetResultType()) {
3009    case Primitive::kPrimInt:
3010      locations->SetInAt(0, Location::RequiresRegister());
3011      locations->SetInAt(1, Location::Any());
3012      if (mul->InputAt(1)->IsIntConstant()) {
3013        // Can use 3 operand multiply.
3014        locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3015      } else {
3016        locations->SetOut(Location::SameAsFirstInput());
3017      }
3018      break;
3019    case Primitive::kPrimLong: {
3020      locations->SetInAt(0, Location::RequiresRegister());
3021      locations->SetInAt(1, Location::Any());
3022      locations->SetOut(Location::SameAsFirstInput());
3023      // Needed for imul on 32bits with 64bits output.
3024      locations->AddTemp(Location::RegisterLocation(EAX));
3025      locations->AddTemp(Location::RegisterLocation(EDX));
3026      break;
3027    }
3028    case Primitive::kPrimFloat:
3029    case Primitive::kPrimDouble: {
3030      locations->SetInAt(0, Location::RequiresFpuRegister());
3031      if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
3032        DCHECK(mul->InputAt(1)->IsEmittedAtUseSite());
3033      } else if (mul->InputAt(1)->IsConstant()) {
3034        locations->SetInAt(1, Location::RequiresFpuRegister());
3035      } else {
3036        locations->SetInAt(1, Location::Any());
3037      }
3038      locations->SetOut(Location::SameAsFirstInput());
3039      break;
3040    }
3041
3042    default:
3043      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
3044  }
3045}
3046
3047void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
3048  LocationSummary* locations = mul->GetLocations();
3049  Location first = locations->InAt(0);
3050  Location second = locations->InAt(1);
3051  Location out = locations->Out();
3052
3053  switch (mul->GetResultType()) {
3054    case Primitive::kPrimInt:
3055      // The constant may have ended up in a register, so test explicitly to avoid
3056      // problems where the output may not be the same as the first operand.
3057      if (mul->InputAt(1)->IsIntConstant()) {
3058        Immediate imm(mul->InputAt(1)->AsIntConstant()->GetValue());
3059        __ imull(out.AsRegister<Register>(), first.AsRegister<Register>(), imm);
3060      } else if (second.IsRegister()) {
3061        DCHECK(first.Equals(out));
3062        __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
3063      } else {
3064        DCHECK(second.IsStackSlot());
3065        DCHECK(first.Equals(out));
3066        __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
3067      }
3068      break;
3069
3070    case Primitive::kPrimLong: {
3071      Register in1_hi = first.AsRegisterPairHigh<Register>();
3072      Register in1_lo = first.AsRegisterPairLow<Register>();
3073      Register eax = locations->GetTemp(0).AsRegister<Register>();
3074      Register edx = locations->GetTemp(1).AsRegister<Register>();
3075
3076      DCHECK_EQ(EAX, eax);
3077      DCHECK_EQ(EDX, edx);
3078
3079      // input: in1 - 64 bits, in2 - 64 bits.
3080      // output: in1
3081      // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
3082      // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
3083      // parts: in1.lo = (in1.lo * in2.lo)[31:0]
3084      if (second.IsConstant()) {
3085        DCHECK(second.GetConstant()->IsLongConstant());
3086
3087        int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
3088        int32_t low_value = Low32Bits(value);
3089        int32_t high_value = High32Bits(value);
3090        Immediate low(low_value);
3091        Immediate high(high_value);
3092
3093        __ movl(eax, high);
3094        // eax <- in1.lo * in2.hi
3095        __ imull(eax, in1_lo);
3096        // in1.hi <- in1.hi * in2.lo
3097        __ imull(in1_hi, low);
3098        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
3099        __ addl(in1_hi, eax);
3100        // move in2_lo to eax to prepare for double precision
3101        __ movl(eax, low);
3102        // edx:eax <- in1.lo * in2.lo
3103        __ mull(in1_lo);
3104        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
3105        __ addl(in1_hi, edx);
3106        // in1.lo <- (in1.lo * in2.lo)[31:0];
3107        __ movl(in1_lo, eax);
3108      } else if (second.IsRegisterPair()) {
3109        Register in2_hi = second.AsRegisterPairHigh<Register>();
3110        Register in2_lo = second.AsRegisterPairLow<Register>();
3111
3112        __ movl(eax, in2_hi);
3113        // eax <- in1.lo * in2.hi
3114        __ imull(eax, in1_lo);
3115        // in1.hi <- in1.hi * in2.lo
3116        __ imull(in1_hi, in2_lo);
3117        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
3118        __ addl(in1_hi, eax);
3119        // move in1_lo to eax to prepare for double precision
3120        __ movl(eax, in1_lo);
3121        // edx:eax <- in1.lo * in2.lo
3122        __ mull(in2_lo);
3123        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
3124        __ addl(in1_hi, edx);
3125        // in1.lo <- (in1.lo * in2.lo)[31:0];
3126        __ movl(in1_lo, eax);
3127      } else {
3128        DCHECK(second.IsDoubleStackSlot()) << second;
3129        Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
3130        Address in2_lo(ESP, second.GetStackIndex());
3131
3132        __ movl(eax, in2_hi);
3133        // eax <- in1.lo * in2.hi
3134        __ imull(eax, in1_lo);
3135        // in1.hi <- in1.hi * in2.lo
3136        __ imull(in1_hi, in2_lo);
3137        // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
3138        __ addl(in1_hi, eax);
3139        // move in1_lo to eax to prepare for double precision
3140        __ movl(eax, in1_lo);
3141        // edx:eax <- in1.lo * in2.lo
3142        __ mull(in2_lo);
3143        // in1.hi <- in2.hi * in1.lo +  in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
3144        __ addl(in1_hi, edx);
3145        // in1.lo <- (in1.lo * in2.lo)[31:0];
3146        __ movl(in1_lo, eax);
3147      }
3148
3149      break;
3150    }
3151
3152    case Primitive::kPrimFloat: {
3153      DCHECK(first.Equals(locations->Out()));
3154      if (second.IsFpuRegister()) {
3155        __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3156      } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
3157        HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
3158        DCHECK(const_area->IsEmittedAtUseSite());
3159        __ mulss(first.AsFpuRegister<XmmRegister>(),
3160                 codegen_->LiteralFloatAddress(
3161                     const_area->GetConstant()->AsFloatConstant()->GetValue(),
3162                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3163      } else {
3164        DCHECK(second.IsStackSlot());
3165        __ mulss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3166      }
3167      break;
3168    }
3169
3170    case Primitive::kPrimDouble: {
3171      DCHECK(first.Equals(locations->Out()));
3172      if (second.IsFpuRegister()) {
3173        __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3174      } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
3175        HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
3176        DCHECK(const_area->IsEmittedAtUseSite());
3177        __ mulsd(first.AsFpuRegister<XmmRegister>(),
3178                 codegen_->LiteralDoubleAddress(
3179                     const_area->GetConstant()->AsDoubleConstant()->GetValue(),
3180                     const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3181      } else {
3182        DCHECK(second.IsDoubleStackSlot());
3183        __ mulsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3184      }
3185      break;
3186    }
3187
3188    default:
3189      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
3190  }
3191}
3192
3193void InstructionCodeGeneratorX86::PushOntoFPStack(Location source,
3194                                                  uint32_t temp_offset,
3195                                                  uint32_t stack_adjustment,
3196                                                  bool is_fp,
3197                                                  bool is_wide) {
3198  if (source.IsStackSlot()) {
3199    DCHECK(!is_wide);
3200    if (is_fp) {
3201      __ flds(Address(ESP, source.GetStackIndex() + stack_adjustment));
3202    } else {
3203      __ filds(Address(ESP, source.GetStackIndex() + stack_adjustment));
3204    }
3205  } else if (source.IsDoubleStackSlot()) {
3206    DCHECK(is_wide);
3207    if (is_fp) {
3208      __ fldl(Address(ESP, source.GetStackIndex() + stack_adjustment));
3209    } else {
3210      __ fildl(Address(ESP, source.GetStackIndex() + stack_adjustment));
3211    }
3212  } else {
3213    // Write the value to the temporary location on the stack and load to FP stack.
3214    if (!is_wide) {
3215      Location stack_temp = Location::StackSlot(temp_offset);
3216      codegen_->Move32(stack_temp, source);
3217      if (is_fp) {
3218        __ flds(Address(ESP, temp_offset));
3219      } else {
3220        __ filds(Address(ESP, temp_offset));
3221      }
3222    } else {
3223      Location stack_temp = Location::DoubleStackSlot(temp_offset);
3224      codegen_->Move64(stack_temp, source);
3225      if (is_fp) {
3226        __ fldl(Address(ESP, temp_offset));
3227      } else {
3228        __ fildl(Address(ESP, temp_offset));
3229      }
3230    }
3231  }
3232}
3233
3234void InstructionCodeGeneratorX86::GenerateRemFP(HRem *rem) {
3235  Primitive::Type type = rem->GetResultType();
3236  bool is_float = type == Primitive::kPrimFloat;
3237  size_t elem_size = Primitive::ComponentSize(type);
3238  LocationSummary* locations = rem->GetLocations();
3239  Location first = locations->InAt(0);
3240  Location second = locations->InAt(1);
3241  Location out = locations->Out();
3242
3243  // Create stack space for 2 elements.
3244  // TODO: enhance register allocator to ask for stack temporaries.
3245  __ subl(ESP, Immediate(2 * elem_size));
3246
3247  // Load the values to the FP stack in reverse order, using temporaries if needed.
3248  const bool is_wide = !is_float;
3249  PushOntoFPStack(second, elem_size, 2 * elem_size, /* is_fp */ true, is_wide);
3250  PushOntoFPStack(first, 0, 2 * elem_size, /* is_fp */ true, is_wide);
3251
3252  // Loop doing FPREM until we stabilize.
3253  NearLabel retry;
3254  __ Bind(&retry);
3255  __ fprem();
3256
3257  // Move FP status to AX.
3258  __ fstsw();
3259
3260  // And see if the argument reduction is complete. This is signaled by the
3261  // C2 FPU flag bit set to 0.
3262  __ andl(EAX, Immediate(kC2ConditionMask));
3263  __ j(kNotEqual, &retry);
3264
3265  // We have settled on the final value. Retrieve it into an XMM register.
3266  // Store FP top of stack to real stack.
3267  if (is_float) {
3268    __ fsts(Address(ESP, 0));
3269  } else {
3270    __ fstl(Address(ESP, 0));
3271  }
3272
3273  // Pop the 2 items from the FP stack.
3274  __ fucompp();
3275
3276  // Load the value from the stack into an XMM register.
3277  DCHECK(out.IsFpuRegister()) << out;
3278  if (is_float) {
3279    __ movss(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
3280  } else {
3281    __ movsd(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
3282  }
3283
3284  // And remove the temporary stack space we allocated.
3285  __ addl(ESP, Immediate(2 * elem_size));
3286}
3287
3288
3289void InstructionCodeGeneratorX86::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
3290  DCHECK(instruction->IsDiv() || instruction->IsRem());
3291
3292  LocationSummary* locations = instruction->GetLocations();
3293  DCHECK(locations->InAt(1).IsConstant());
3294  DCHECK(locations->InAt(1).GetConstant()->IsIntConstant());
3295
3296  Register out_register = locations->Out().AsRegister<Register>();
3297  Register input_register = locations->InAt(0).AsRegister<Register>();
3298  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3299
3300  DCHECK(imm == 1 || imm == -1);
3301
3302  if (instruction->IsRem()) {
3303    __ xorl(out_register, out_register);
3304  } else {
3305    __ movl(out_register, input_register);
3306    if (imm == -1) {
3307      __ negl(out_register);
3308    }
3309  }
3310}
3311
3312
3313void InstructionCodeGeneratorX86::DivByPowerOfTwo(HDiv* instruction) {
3314  LocationSummary* locations = instruction->GetLocations();
3315
3316  Register out_register = locations->Out().AsRegister<Register>();
3317  Register input_register = locations->InAt(0).AsRegister<Register>();
3318  int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3319  DCHECK(IsPowerOfTwo(AbsOrMin(imm)));
3320  uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm));
3321
3322  Register num = locations->GetTemp(0).AsRegister<Register>();
3323
3324  __ leal(num, Address(input_register, abs_imm - 1));
3325  __ testl(input_register, input_register);
3326  __ cmovl(kGreaterEqual, num, input_register);
3327  int shift = CTZ(imm);
3328  __ sarl(num, Immediate(shift));
3329
3330  if (imm < 0) {
3331    __ negl(num);
3332  }
3333
3334  __ movl(out_register, num);
3335}
3336
3337void InstructionCodeGeneratorX86::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
3338  DCHECK(instruction->IsDiv() || instruction->IsRem());
3339
3340  LocationSummary* locations = instruction->GetLocations();
3341  int imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3342
3343  Register eax = locations->InAt(0).AsRegister<Register>();
3344  Register out = locations->Out().AsRegister<Register>();
3345  Register num;
3346  Register edx;
3347
3348  if (instruction->IsDiv()) {
3349    edx = locations->GetTemp(0).AsRegister<Register>();
3350    num = locations->GetTemp(1).AsRegister<Register>();
3351  } else {
3352    edx = locations->Out().AsRegister<Register>();
3353    num = locations->GetTemp(0).AsRegister<Register>();
3354  }
3355
3356  DCHECK_EQ(EAX, eax);
3357  DCHECK_EQ(EDX, edx);
3358  if (instruction->IsDiv()) {
3359    DCHECK_EQ(EAX, out);
3360  } else {
3361    DCHECK_EQ(EDX, out);
3362  }
3363
3364  int64_t magic;
3365  int shift;
3366  CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
3367
3368  NearLabel ndiv;
3369  NearLabel end;
3370  // If numerator is 0, the result is 0, no computation needed.
3371  __ testl(eax, eax);
3372  __ j(kNotEqual, &ndiv);
3373
3374  __ xorl(out, out);
3375  __ jmp(&end);
3376
3377  __ Bind(&ndiv);
3378
3379  // Save the numerator.
3380  __ movl(num, eax);
3381
3382  // EAX = magic
3383  __ movl(eax, Immediate(magic));
3384
3385  // EDX:EAX = magic * numerator
3386  __ imull(num);
3387
3388  if (imm > 0 && magic < 0) {
3389    // EDX += num
3390    __ addl(edx, num);
3391  } else if (imm < 0 && magic > 0) {
3392    __ subl(edx, num);
3393  }
3394
3395  // Shift if needed.
3396  if (shift != 0) {
3397    __ sarl(edx, Immediate(shift));
3398  }
3399
3400  // EDX += 1 if EDX < 0
3401  __ movl(eax, edx);
3402  __ shrl(edx, Immediate(31));
3403  __ addl(edx, eax);
3404
3405  if (instruction->IsRem()) {
3406    __ movl(eax, num);
3407    __ imull(edx, Immediate(imm));
3408    __ subl(eax, edx);
3409    __ movl(edx, eax);
3410  } else {
3411    __ movl(eax, edx);
3412  }
3413  __ Bind(&end);
3414}
3415
3416void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
3417  DCHECK(instruction->IsDiv() || instruction->IsRem());
3418
3419  LocationSummary* locations = instruction->GetLocations();
3420  Location out = locations->Out();
3421  Location first = locations->InAt(0);
3422  Location second = locations->InAt(1);
3423  bool is_div = instruction->IsDiv();
3424
3425  switch (instruction->GetResultType()) {
3426    case Primitive::kPrimInt: {
3427      DCHECK_EQ(EAX, first.AsRegister<Register>());
3428      DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
3429
3430      if (second.IsConstant()) {
3431        int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
3432
3433        if (imm == 0) {
3434          // Do not generate anything for 0. DivZeroCheck would forbid any generated code.
3435        } else if (imm == 1 || imm == -1) {
3436          DivRemOneOrMinusOne(instruction);
3437        } else if (is_div && IsPowerOfTwo(AbsOrMin(imm))) {
3438          DivByPowerOfTwo(instruction->AsDiv());
3439        } else {
3440          DCHECK(imm <= -2 || imm >= 2);
3441          GenerateDivRemWithAnyConstant(instruction);
3442        }
3443      } else {
3444        SlowPathCode* slow_path = new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(
3445            instruction, out.AsRegister<Register>(), is_div);
3446        codegen_->AddSlowPath(slow_path);
3447
3448        Register second_reg = second.AsRegister<Register>();
3449        // 0x80000000/-1 triggers an arithmetic exception!
3450        // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
3451        // it's safe to just use negl instead of more complex comparisons.
3452
3453        __ cmpl(second_reg, Immediate(-1));
3454        __ j(kEqual, slow_path->GetEntryLabel());
3455
3456        // edx:eax <- sign-extended of eax
3457        __ cdq();
3458        // eax = quotient, edx = remainder
3459        __ idivl(second_reg);
3460        __ Bind(slow_path->GetExitLabel());
3461      }
3462      break;
3463    }
3464
3465    case Primitive::kPrimLong: {
3466      InvokeRuntimeCallingConvention calling_convention;
3467      DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
3468      DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
3469      DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
3470      DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
3471      DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
3472      DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
3473
3474      if (is_div) {
3475        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLdiv),
3476                                instruction,
3477                                instruction->GetDexPc(),
3478                                nullptr);
3479        CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>();
3480      } else {
3481        codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLmod),
3482                                instruction,
3483                                instruction->GetDexPc(),
3484                                nullptr);
3485        CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>();
3486      }
3487      break;
3488    }
3489
3490    default:
3491      LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
3492  }
3493}
3494
3495void LocationsBuilderX86::VisitDiv(HDiv* div) {
3496  LocationSummary::CallKind call_kind = (div->GetResultType() == Primitive::kPrimLong)
3497      ? LocationSummary::kCall
3498      : LocationSummary::kNoCall;
3499  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
3500
3501  switch (div->GetResultType()) {
3502    case Primitive::kPrimInt: {
3503      locations->SetInAt(0, Location::RegisterLocation(EAX));
3504      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
3505      locations->SetOut(Location::SameAsFirstInput());
3506      // Intel uses edx:eax as the dividend.
3507      locations->AddTemp(Location::RegisterLocation(EDX));
3508      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
3509      // which enforces results to be in EAX and EDX, things are simpler if we use EAX also as
3510      // output and request another temp.
3511      if (div->InputAt(1)->IsIntConstant()) {
3512        locations->AddTemp(Location::RequiresRegister());
3513      }
3514      break;
3515    }
3516    case Primitive::kPrimLong: {
3517      InvokeRuntimeCallingConvention calling_convention;
3518      locations->SetInAt(0, Location::RegisterPairLocation(
3519          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
3520      locations->SetInAt(1, Location::RegisterPairLocation(
3521          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
3522      // Runtime helper puts the result in EAX, EDX.
3523      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
3524      break;
3525    }
3526    case Primitive::kPrimFloat:
3527    case Primitive::kPrimDouble: {
3528      locations->SetInAt(0, Location::RequiresFpuRegister());
3529      if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3530        DCHECK(div->InputAt(1)->IsEmittedAtUseSite());
3531      } else if (div->InputAt(1)->IsConstant()) {
3532        locations->SetInAt(1, Location::RequiresFpuRegister());
3533      } else {
3534        locations->SetInAt(1, Location::Any());
3535      }
3536      locations->SetOut(Location::SameAsFirstInput());
3537      break;
3538    }
3539
3540    default:
3541      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
3542  }
3543}
3544
3545void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
3546  LocationSummary* locations = div->GetLocations();
3547  Location first = locations->InAt(0);
3548  Location second = locations->InAt(1);
3549
3550  switch (div->GetResultType()) {
3551    case Primitive::kPrimInt:
3552    case Primitive::kPrimLong: {
3553      GenerateDivRemIntegral(div);
3554      break;
3555    }
3556
3557    case Primitive::kPrimFloat: {
3558      if (second.IsFpuRegister()) {
3559        __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3560      } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3561        HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3562        DCHECK(const_area->IsEmittedAtUseSite());
3563        __ divss(first.AsFpuRegister<XmmRegister>(),
3564                 codegen_->LiteralFloatAddress(
3565                   const_area->GetConstant()->AsFloatConstant()->GetValue(),
3566                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3567      } else {
3568        DCHECK(second.IsStackSlot());
3569        __ divss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3570      }
3571      break;
3572    }
3573
3574    case Primitive::kPrimDouble: {
3575      if (second.IsFpuRegister()) {
3576        __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3577      } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3578        HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3579        DCHECK(const_area->IsEmittedAtUseSite());
3580        __ divsd(first.AsFpuRegister<XmmRegister>(),
3581                 codegen_->LiteralDoubleAddress(
3582                   const_area->GetConstant()->AsDoubleConstant()->GetValue(),
3583                   const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3584      } else {
3585        DCHECK(second.IsDoubleStackSlot());
3586        __ divsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3587      }
3588      break;
3589    }
3590
3591    default:
3592      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
3593  }
3594}
3595
3596void LocationsBuilderX86::VisitRem(HRem* rem) {
3597  Primitive::Type type = rem->GetResultType();
3598
3599  LocationSummary::CallKind call_kind = (rem->GetResultType() == Primitive::kPrimLong)
3600      ? LocationSummary::kCall
3601      : LocationSummary::kNoCall;
3602  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
3603
3604  switch (type) {
3605    case Primitive::kPrimInt: {
3606      locations->SetInAt(0, Location::RegisterLocation(EAX));
3607      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
3608      locations->SetOut(Location::RegisterLocation(EDX));
3609      // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
3610      // which enforces results to be in EAX and EDX, things are simpler if we use EDX also as
3611      // output and request another temp.
3612      if (rem->InputAt(1)->IsIntConstant()) {
3613        locations->AddTemp(Location::RequiresRegister());
3614      }
3615      break;
3616    }
3617    case Primitive::kPrimLong: {
3618      InvokeRuntimeCallingConvention calling_convention;
3619      locations->SetInAt(0, Location::RegisterPairLocation(
3620          calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
3621      locations->SetInAt(1, Location::RegisterPairLocation(
3622          calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
3623      // Runtime helper puts the result in EAX, EDX.
3624      locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
3625      break;
3626    }
3627    case Primitive::kPrimDouble:
3628    case Primitive::kPrimFloat: {
3629      locations->SetInAt(0, Location::Any());
3630      locations->SetInAt(1, Location::Any());
3631      locations->SetOut(Location::RequiresFpuRegister());
3632      locations->AddTemp(Location::RegisterLocation(EAX));
3633      break;
3634    }
3635
3636    default:
3637      LOG(FATAL) << "Unexpected rem type " << type;
3638  }
3639}
3640
3641void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
3642  Primitive::Type type = rem->GetResultType();
3643  switch (type) {
3644    case Primitive::kPrimInt:
3645    case Primitive::kPrimLong: {
3646      GenerateDivRemIntegral(rem);
3647      break;
3648    }
3649    case Primitive::kPrimFloat:
3650    case Primitive::kPrimDouble: {
3651      GenerateRemFP(rem);
3652      break;
3653    }
3654    default:
3655      LOG(FATAL) << "Unexpected rem type " << type;
3656  }
3657}
3658
3659void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3660  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
3661      ? LocationSummary::kCallOnSlowPath
3662      : LocationSummary::kNoCall;
3663  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3664  switch (instruction->GetType()) {
3665    case Primitive::kPrimBoolean:
3666    case Primitive::kPrimByte:
3667    case Primitive::kPrimChar:
3668    case Primitive::kPrimShort:
3669    case Primitive::kPrimInt: {
3670      locations->SetInAt(0, Location::Any());
3671      break;
3672    }
3673    case Primitive::kPrimLong: {
3674      locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3675      if (!instruction->IsConstant()) {
3676        locations->AddTemp(Location::RequiresRegister());
3677      }
3678      break;
3679    }
3680    default:
3681      LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
3682  }
3683  if (instruction->HasUses()) {
3684    locations->SetOut(Location::SameAsFirstInput());
3685  }
3686}
3687
3688void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3689  SlowPathCode* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
3690  codegen_->AddSlowPath(slow_path);
3691
3692  LocationSummary* locations = instruction->GetLocations();
3693  Location value = locations->InAt(0);
3694
3695  switch (instruction->GetType()) {
3696    case Primitive::kPrimBoolean:
3697    case Primitive::kPrimByte:
3698    case Primitive::kPrimChar:
3699    case Primitive::kPrimShort:
3700    case Primitive::kPrimInt: {
3701      if (value.IsRegister()) {
3702        __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
3703        __ j(kEqual, slow_path->GetEntryLabel());
3704      } else if (value.IsStackSlot()) {
3705        __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
3706        __ j(kEqual, slow_path->GetEntryLabel());
3707      } else {
3708        DCHECK(value.IsConstant()) << value;
3709        if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
3710        __ jmp(slow_path->GetEntryLabel());
3711        }
3712      }
3713      break;
3714    }
3715    case Primitive::kPrimLong: {
3716      if (value.IsRegisterPair()) {
3717        Register temp = locations->GetTemp(0).AsRegister<Register>();
3718        __ movl(temp, value.AsRegisterPairLow<Register>());
3719        __ orl(temp, value.AsRegisterPairHigh<Register>());
3720        __ j(kEqual, slow_path->GetEntryLabel());
3721      } else {
3722        DCHECK(value.IsConstant()) << value;
3723        if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
3724          __ jmp(slow_path->GetEntryLabel());
3725        }
3726      }
3727      break;
3728    }
3729    default:
3730      LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
3731  }
3732}
3733
3734void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
3735  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3736
3737  LocationSummary* locations =
3738      new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
3739
3740  switch (op->GetResultType()) {
3741    case Primitive::kPrimInt:
3742    case Primitive::kPrimLong: {
3743      // Can't have Location::Any() and output SameAsFirstInput()
3744      locations->SetInAt(0, Location::RequiresRegister());
3745      // The shift count needs to be in CL or a constant.
3746      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
3747      locations->SetOut(Location::SameAsFirstInput());
3748      break;
3749    }
3750    default:
3751      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3752  }
3753}
3754
3755void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
3756  DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3757
3758  LocationSummary* locations = op->GetLocations();
3759  Location first = locations->InAt(0);
3760  Location second = locations->InAt(1);
3761  DCHECK(first.Equals(locations->Out()));
3762
3763  switch (op->GetResultType()) {
3764    case Primitive::kPrimInt: {
3765      DCHECK(first.IsRegister());
3766      Register first_reg = first.AsRegister<Register>();
3767      if (second.IsRegister()) {
3768        Register second_reg = second.AsRegister<Register>();
3769        DCHECK_EQ(ECX, second_reg);
3770        if (op->IsShl()) {
3771          __ shll(first_reg, second_reg);
3772        } else if (op->IsShr()) {
3773          __ sarl(first_reg, second_reg);
3774        } else {
3775          __ shrl(first_reg, second_reg);
3776        }
3777      } else {
3778        int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue;
3779        if (shift == 0) {
3780          return;
3781        }
3782        Immediate imm(shift);
3783        if (op->IsShl()) {
3784          __ shll(first_reg, imm);
3785        } else if (op->IsShr()) {
3786          __ sarl(first_reg, imm);
3787        } else {
3788          __ shrl(first_reg, imm);
3789        }
3790      }
3791      break;
3792    }
3793    case Primitive::kPrimLong: {
3794      if (second.IsRegister()) {
3795        Register second_reg = second.AsRegister<Register>();
3796        DCHECK_EQ(ECX, second_reg);
3797        if (op->IsShl()) {
3798          GenerateShlLong(first, second_reg);
3799        } else if (op->IsShr()) {
3800          GenerateShrLong(first, second_reg);
3801        } else {
3802          GenerateUShrLong(first, second_reg);
3803        }
3804      } else {
3805        // Shift by a constant.
3806        int shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue;
3807        // Nothing to do if the shift is 0, as the input is already the output.
3808        if (shift != 0) {
3809          if (op->IsShl()) {
3810            GenerateShlLong(first, shift);
3811          } else if (op->IsShr()) {
3812            GenerateShrLong(first, shift);
3813          } else {
3814            GenerateUShrLong(first, shift);
3815          }
3816        }
3817      }
3818      break;
3819    }
3820    default:
3821      LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3822  }
3823}
3824
3825void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, int shift) {
3826  Register low = loc.AsRegisterPairLow<Register>();
3827  Register high = loc.AsRegisterPairHigh<Register>();
3828  if (shift == 1) {
3829    // This is just an addition.
3830    __ addl(low, low);
3831    __ adcl(high, high);
3832  } else if (shift == 32) {
3833    // Shift by 32 is easy. High gets low, and low gets 0.
3834    codegen_->EmitParallelMoves(
3835        loc.ToLow(),
3836        loc.ToHigh(),
3837        Primitive::kPrimInt,
3838        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3839        loc.ToLow(),
3840        Primitive::kPrimInt);
3841  } else if (shift > 32) {
3842    // Low part becomes 0.  High part is low part << (shift-32).
3843    __ movl(high, low);
3844    __ shll(high, Immediate(shift - 32));
3845    __ xorl(low, low);
3846  } else {
3847    // Between 1 and 31.
3848    __ shld(high, low, Immediate(shift));
3849    __ shll(low, Immediate(shift));
3850  }
3851}
3852
3853void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
3854  NearLabel done;
3855  __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
3856  __ shll(loc.AsRegisterPairLow<Register>(), shifter);
3857  __ testl(shifter, Immediate(32));
3858  __ j(kEqual, &done);
3859  __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
3860  __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
3861  __ Bind(&done);
3862}
3863
3864void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, int shift) {
3865  Register low = loc.AsRegisterPairLow<Register>();
3866  Register high = loc.AsRegisterPairHigh<Register>();
3867  if (shift == 32) {
3868    // Need to copy the sign.
3869    DCHECK_NE(low, high);
3870    __ movl(low, high);
3871    __ sarl(high, Immediate(31));
3872  } else if (shift > 32) {
3873    DCHECK_NE(low, high);
3874    // High part becomes sign. Low part is shifted by shift - 32.
3875    __ movl(low, high);
3876    __ sarl(high, Immediate(31));
3877    __ sarl(low, Immediate(shift - 32));
3878  } else {
3879    // Between 1 and 31.
3880    __ shrd(low, high, Immediate(shift));
3881    __ sarl(high, Immediate(shift));
3882  }
3883}
3884
3885void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
3886  NearLabel done;
3887  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3888  __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
3889  __ testl(shifter, Immediate(32));
3890  __ j(kEqual, &done);
3891  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3892  __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
3893  __ Bind(&done);
3894}
3895
3896void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, int shift) {
3897  Register low = loc.AsRegisterPairLow<Register>();
3898  Register high = loc.AsRegisterPairHigh<Register>();
3899  if (shift == 32) {
3900    // Shift by 32 is easy. Low gets high, and high gets 0.
3901    codegen_->EmitParallelMoves(
3902        loc.ToHigh(),
3903        loc.ToLow(),
3904        Primitive::kPrimInt,
3905        Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3906        loc.ToHigh(),
3907        Primitive::kPrimInt);
3908  } else if (shift > 32) {
3909    // Low part is high >> (shift - 32). High part becomes 0.
3910    __ movl(low, high);
3911    __ shrl(low, Immediate(shift - 32));
3912    __ xorl(high, high);
3913  } else {
3914    // Between 1 and 31.
3915    __ shrd(low, high, Immediate(shift));
3916    __ shrl(high, Immediate(shift));
3917  }
3918}
3919
3920void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
3921  NearLabel done;
3922  __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3923  __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
3924  __ testl(shifter, Immediate(32));
3925  __ j(kEqual, &done);
3926  __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3927  __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
3928  __ Bind(&done);
3929}
3930
3931void LocationsBuilderX86::VisitRor(HRor* ror) {
3932  LocationSummary* locations =
3933      new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall);
3934
3935  switch (ror->GetResultType()) {
3936    case Primitive::kPrimLong:
3937      // Add the temporary needed.
3938      locations->AddTemp(Location::RequiresRegister());
3939      FALLTHROUGH_INTENDED;
3940    case Primitive::kPrimInt:
3941      locations->SetInAt(0, Location::RequiresRegister());
3942      // The shift count needs to be in CL (unless it is a constant).
3943      locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, ror->InputAt(1)));
3944      locations->SetOut(Location::SameAsFirstInput());
3945      break;
3946    default:
3947      LOG(FATAL) << "Unexpected operation type " << ror->GetResultType();
3948      UNREACHABLE();
3949  }
3950}
3951
3952void InstructionCodeGeneratorX86::VisitRor(HRor* ror) {
3953  LocationSummary* locations = ror->GetLocations();
3954  Location first = locations->InAt(0);
3955  Location second = locations->InAt(1);
3956
3957  if (ror->GetResultType() == Primitive::kPrimInt) {
3958    Register first_reg = first.AsRegister<Register>();
3959    if (second.IsRegister()) {
3960      Register second_reg = second.AsRegister<Register>();
3961      __ rorl(first_reg, second_reg);
3962    } else {
3963      Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue);
3964      __ rorl(first_reg, imm);
3965    }
3966    return;
3967  }
3968
3969  DCHECK_EQ(ror->GetResultType(), Primitive::kPrimLong);
3970  Register first_reg_lo = first.AsRegisterPairLow<Register>();
3971  Register first_reg_hi = first.AsRegisterPairHigh<Register>();
3972  Register temp_reg = locations->GetTemp(0).AsRegister<Register>();
3973  if (second.IsRegister()) {
3974    Register second_reg = second.AsRegister<Register>();
3975    DCHECK_EQ(second_reg, ECX);
3976    __ movl(temp_reg, first_reg_hi);
3977    __ shrd(first_reg_hi, first_reg_lo, second_reg);
3978    __ shrd(first_reg_lo, temp_reg, second_reg);
3979    __ movl(temp_reg, first_reg_hi);
3980    __ testl(second_reg, Immediate(32));
3981    __ cmovl(kNotEqual, first_reg_hi, first_reg_lo);
3982    __ cmovl(kNotEqual, first_reg_lo, temp_reg);
3983  } else {
3984    int32_t shift_amt = CodeGenerator::GetInt64ValueOf(second.GetConstant()) & kMaxLongShiftValue;
3985    if (shift_amt == 0) {
3986      // Already fine.
3987      return;
3988    }
3989    if (shift_amt == 32) {
3990      // Just swap.
3991      __ movl(temp_reg, first_reg_lo);
3992      __ movl(first_reg_lo, first_reg_hi);
3993      __ movl(first_reg_hi, temp_reg);
3994      return;
3995    }
3996
3997    Immediate imm(shift_amt);
3998    // Save the constents of the low value.
3999    __ movl(temp_reg, first_reg_lo);
4000
4001    // Shift right into low, feeding bits from high.
4002    __ shrd(first_reg_lo, first_reg_hi, imm);
4003
4004    // Shift right into high, feeding bits from the original low.
4005    __ shrd(first_reg_hi, temp_reg, imm);
4006
4007    // Swap if needed.
4008    if (shift_amt > 32) {
4009      __ movl(temp_reg, first_reg_lo);
4010      __ movl(first_reg_lo, first_reg_hi);
4011      __ movl(first_reg_hi, temp_reg);
4012    }
4013  }
4014}
4015
4016void LocationsBuilderX86::VisitShl(HShl* shl) {
4017  HandleShift(shl);
4018}
4019
4020void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
4021  HandleShift(shl);
4022}
4023
4024void LocationsBuilderX86::VisitShr(HShr* shr) {
4025  HandleShift(shr);
4026}
4027
4028void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
4029  HandleShift(shr);
4030}
4031
4032void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
4033  HandleShift(ushr);
4034}
4035
4036void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
4037  HandleShift(ushr);
4038}
4039
4040void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
4041  LocationSummary* locations =
4042      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4043  locations->SetOut(Location::RegisterLocation(EAX));
4044  if (instruction->IsStringAlloc()) {
4045    locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument));
4046  } else {
4047    InvokeRuntimeCallingConvention calling_convention;
4048    locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4049    locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
4050  }
4051}
4052
4053void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
4054  // Note: if heap poisoning is enabled, the entry point takes cares
4055  // of poisoning the reference.
4056  if (instruction->IsStringAlloc()) {
4057    // String is allocated through StringFactory. Call NewEmptyString entry point.
4058    Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>();
4059    MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize);
4060    __ fs()->movl(temp, Address::Absolute(QUICK_ENTRY_POINT(pNewEmptyString)));
4061    __ call(Address(temp, code_offset.Int32Value()));
4062    codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4063  } else {
4064    codegen_->InvokeRuntime(instruction->GetEntrypoint(),
4065                            instruction,
4066                            instruction->GetDexPc(),
4067                            nullptr);
4068    CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
4069    DCHECK(!codegen_->IsLeafMethod());
4070  }
4071}
4072
4073void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
4074  LocationSummary* locations =
4075      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4076  locations->SetOut(Location::RegisterLocation(EAX));
4077  InvokeRuntimeCallingConvention calling_convention;
4078  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4079  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
4080  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
4081}
4082
4083void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
4084  InvokeRuntimeCallingConvention calling_convention;
4085  __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
4086  // Note: if heap poisoning is enabled, the entry point takes cares
4087  // of poisoning the reference.
4088  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
4089                          instruction,
4090                          instruction->GetDexPc(),
4091                          nullptr);
4092  CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>();
4093  DCHECK(!codegen_->IsLeafMethod());
4094}
4095
4096void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
4097  LocationSummary* locations =
4098      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4099  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
4100  if (location.IsStackSlot()) {
4101    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
4102  } else if (location.IsDoubleStackSlot()) {
4103    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
4104  }
4105  locations->SetOut(location);
4106}
4107
4108void InstructionCodeGeneratorX86::VisitParameterValue(
4109    HParameterValue* instruction ATTRIBUTE_UNUSED) {
4110}
4111
4112void LocationsBuilderX86::VisitCurrentMethod(HCurrentMethod* instruction) {
4113  LocationSummary* locations =
4114      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4115  locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument));
4116}
4117
4118void InstructionCodeGeneratorX86::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
4119}
4120
4121void LocationsBuilderX86::VisitClassTableGet(HClassTableGet* instruction) {
4122  LocationSummary* locations =
4123      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4124  locations->SetInAt(0, Location::RequiresRegister());
4125  locations->SetOut(Location::RequiresRegister());
4126}
4127
4128void InstructionCodeGeneratorX86::VisitClassTableGet(HClassTableGet* instruction) {
4129  LocationSummary* locations = instruction->GetLocations();
4130  uint32_t method_offset = 0;
4131  if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) {
4132    method_offset = mirror::Class::EmbeddedVTableEntryOffset(
4133        instruction->GetIndex(), kX86PointerSize).SizeValue();
4134  } else {
4135    method_offset = mirror::Class::EmbeddedImTableEntryOffset(
4136        instruction->GetIndex() % mirror::Class::kImtSize, kX86PointerSize).Uint32Value();
4137  }
4138  __ movl(locations->Out().AsRegister<Register>(),
4139          Address(locations->InAt(0).AsRegister<Register>(), method_offset));
4140}
4141
4142void LocationsBuilderX86::VisitNot(HNot* not_) {
4143  LocationSummary* locations =
4144      new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
4145  locations->SetInAt(0, Location::RequiresRegister());
4146  locations->SetOut(Location::SameAsFirstInput());
4147}
4148
4149void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
4150  LocationSummary* locations = not_->GetLocations();
4151  Location in = locations->InAt(0);
4152  Location out = locations->Out();
4153  DCHECK(in.Equals(out));
4154  switch (not_->GetResultType()) {
4155    case Primitive::kPrimInt:
4156      __ notl(out.AsRegister<Register>());
4157      break;
4158
4159    case Primitive::kPrimLong:
4160      __ notl(out.AsRegisterPairLow<Register>());
4161      __ notl(out.AsRegisterPairHigh<Register>());
4162      break;
4163
4164    default:
4165      LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
4166  }
4167}
4168
4169void LocationsBuilderX86::VisitBooleanNot(HBooleanNot* bool_not) {
4170  LocationSummary* locations =
4171      new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
4172  locations->SetInAt(0, Location::RequiresRegister());
4173  locations->SetOut(Location::SameAsFirstInput());
4174}
4175
4176void InstructionCodeGeneratorX86::VisitBooleanNot(HBooleanNot* bool_not) {
4177  LocationSummary* locations = bool_not->GetLocations();
4178  Location in = locations->InAt(0);
4179  Location out = locations->Out();
4180  DCHECK(in.Equals(out));
4181  __ xorl(out.AsRegister<Register>(), Immediate(1));
4182}
4183
4184void LocationsBuilderX86::VisitCompare(HCompare* compare) {
4185  LocationSummary* locations =
4186      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
4187  switch (compare->InputAt(0)->GetType()) {
4188    case Primitive::kPrimBoolean:
4189    case Primitive::kPrimByte:
4190    case Primitive::kPrimShort:
4191    case Primitive::kPrimChar:
4192    case Primitive::kPrimInt:
4193    case Primitive::kPrimLong: {
4194      locations->SetInAt(0, Location::RequiresRegister());
4195      locations->SetInAt(1, Location::Any());
4196      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
4197      break;
4198    }
4199    case Primitive::kPrimFloat:
4200    case Primitive::kPrimDouble: {
4201      locations->SetInAt(0, Location::RequiresFpuRegister());
4202      if (compare->InputAt(1)->IsX86LoadFromConstantTable()) {
4203        DCHECK(compare->InputAt(1)->IsEmittedAtUseSite());
4204      } else if (compare->InputAt(1)->IsConstant()) {
4205        locations->SetInAt(1, Location::RequiresFpuRegister());
4206      } else {
4207        locations->SetInAt(1, Location::Any());
4208      }
4209      locations->SetOut(Location::RequiresRegister());
4210      break;
4211    }
4212    default:
4213      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
4214  }
4215}
4216
4217void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
4218  LocationSummary* locations = compare->GetLocations();
4219  Register out = locations->Out().AsRegister<Register>();
4220  Location left = locations->InAt(0);
4221  Location right = locations->InAt(1);
4222
4223  NearLabel less, greater, done;
4224  Condition less_cond = kLess;
4225
4226  switch (compare->InputAt(0)->GetType()) {
4227    case Primitive::kPrimBoolean:
4228    case Primitive::kPrimByte:
4229    case Primitive::kPrimShort:
4230    case Primitive::kPrimChar:
4231    case Primitive::kPrimInt: {
4232      GenerateIntCompare(left, right);
4233      break;
4234    }
4235    case Primitive::kPrimLong: {
4236      Register left_low = left.AsRegisterPairLow<Register>();
4237      Register left_high = left.AsRegisterPairHigh<Register>();
4238      int32_t val_low = 0;
4239      int32_t val_high = 0;
4240      bool right_is_const = false;
4241
4242      if (right.IsConstant()) {
4243        DCHECK(right.GetConstant()->IsLongConstant());
4244        right_is_const = true;
4245        int64_t val = right.GetConstant()->AsLongConstant()->GetValue();
4246        val_low = Low32Bits(val);
4247        val_high = High32Bits(val);
4248      }
4249
4250      if (right.IsRegisterPair()) {
4251        __ cmpl(left_high, right.AsRegisterPairHigh<Register>());
4252      } else if (right.IsDoubleStackSlot()) {
4253        __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
4254      } else {
4255        DCHECK(right_is_const) << right;
4256        codegen_->Compare32BitValue(left_high, val_high);
4257      }
4258      __ j(kLess, &less);  // Signed compare.
4259      __ j(kGreater, &greater);  // Signed compare.
4260      if (right.IsRegisterPair()) {
4261        __ cmpl(left_low, right.AsRegisterPairLow<Register>());
4262      } else if (right.IsDoubleStackSlot()) {
4263        __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
4264      } else {
4265        DCHECK(right_is_const) << right;
4266        codegen_->Compare32BitValue(left_low, val_low);
4267      }
4268      less_cond = kBelow;  // for CF (unsigned).
4269      break;
4270    }
4271    case Primitive::kPrimFloat: {
4272      GenerateFPCompare(left, right, compare, false);
4273      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
4274      less_cond = kBelow;  // for CF (floats).
4275      break;
4276    }
4277    case Primitive::kPrimDouble: {
4278      GenerateFPCompare(left, right, compare, true);
4279      __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
4280      less_cond = kBelow;  // for CF (floats).
4281      break;
4282    }
4283    default:
4284      LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
4285  }
4286
4287  __ movl(out, Immediate(0));
4288  __ j(kEqual, &done);
4289  __ j(less_cond, &less);
4290
4291  __ Bind(&greater);
4292  __ movl(out, Immediate(1));
4293  __ jmp(&done);
4294
4295  __ Bind(&less);
4296  __ movl(out, Immediate(-1));
4297
4298  __ Bind(&done);
4299}
4300
4301void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
4302  LocationSummary* locations =
4303      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4304  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
4305    locations->SetInAt(i, Location::Any());
4306  }
4307  locations->SetOut(Location::Any());
4308}
4309
4310void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) {
4311  LOG(FATAL) << "Unreachable";
4312}
4313
4314void CodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
4315  /*
4316   * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
4317   * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
4318   * For those cases, all we need to ensure is that there is a scheduling barrier in place.
4319   */
4320  switch (kind) {
4321    case MemBarrierKind::kAnyAny: {
4322      MemoryFence();
4323      break;
4324    }
4325    case MemBarrierKind::kAnyStore:
4326    case MemBarrierKind::kLoadAny:
4327    case MemBarrierKind::kStoreStore: {
4328      // nop
4329      break;
4330    }
4331    default:
4332      LOG(FATAL) << "Unexpected memory barrier " << kind;
4333  }
4334}
4335
4336HInvokeStaticOrDirect::DispatchInfo CodeGeneratorX86::GetSupportedInvokeStaticOrDirectDispatch(
4337      const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
4338      MethodReference target_method ATTRIBUTE_UNUSED) {
4339  HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info;
4340
4341  // We disable pc-relative load when there is an irreducible loop, as the optimization
4342  // is incompatible with it.
4343  if (GetGraph()->HasIrreducibleLoops() &&
4344      (dispatch_info.method_load_kind ==
4345          HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) {
4346    dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod;
4347  }
4348  switch (dispatch_info.code_ptr_location) {
4349    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
4350    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
4351      // For direct code, we actually prefer to call via the code pointer from ArtMethod*.
4352      // (Though the direct CALL ptr16:32 is available for consideration).
4353      return HInvokeStaticOrDirect::DispatchInfo {
4354        dispatch_info.method_load_kind,
4355        HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod,
4356        dispatch_info.method_load_data,
4357        0u
4358      };
4359    default:
4360      return dispatch_info;
4361  }
4362}
4363
4364Register CodeGeneratorX86::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke,
4365                                                                 Register temp) {
4366  DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u);
4367  Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4368  if (!invoke->GetLocations()->Intrinsified()) {
4369    return location.AsRegister<Register>();
4370  }
4371  // For intrinsics we allow any location, so it may be on the stack.
4372  if (!location.IsRegister()) {
4373    __ movl(temp, Address(ESP, location.GetStackIndex()));
4374    return temp;
4375  }
4376  // For register locations, check if the register was saved. If so, get it from the stack.
4377  // Note: There is a chance that the register was saved but not overwritten, so we could
4378  // save one load. However, since this is just an intrinsic slow path we prefer this
4379  // simple and more robust approach rather that trying to determine if that's the case.
4380  SlowPathCode* slow_path = GetCurrentSlowPath();
4381  DCHECK(slow_path != nullptr);  // For intrinsified invokes the call is emitted on the slow path.
4382  if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) {
4383    int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>());
4384    __ movl(temp, Address(ESP, stack_offset));
4385    return temp;
4386  }
4387  return location.AsRegister<Register>();
4388}
4389
4390void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) {
4391  Location callee_method = temp;  // For all kinds except kRecursive, callee will be in temp.
4392  switch (invoke->GetMethodLoadKind()) {
4393    case HInvokeStaticOrDirect::MethodLoadKind::kStringInit:
4394      // temp = thread->string_init_entrypoint
4395      __ fs()->movl(temp.AsRegister<Register>(), Address::Absolute(invoke->GetStringInitOffset()));
4396      break;
4397    case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
4398      callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4399      break;
4400    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
4401      __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress()));
4402      break;
4403    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
4404      __ movl(temp.AsRegister<Register>(), Immediate(0));  // Placeholder.
4405      method_patches_.emplace_back(invoke->GetTargetMethod());
4406      __ Bind(&method_patches_.back().label);  // Bind the label at the end of the "movl" insn.
4407      break;
4408    case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
4409      Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke,
4410                                                                temp.AsRegister<Register>());
4411      uint32_t offset = invoke->GetDexCacheArrayOffset();
4412      __ movl(temp.AsRegister<Register>(), Address(base_reg, kDummy32BitOffset));
4413      // Add the patch entry and bind its label at the end of the instruction.
4414      pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, offset);
4415      __ Bind(&pc_relative_dex_cache_patches_.back().label);
4416      break;
4417    }
4418    case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
4419      Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4420      Register method_reg;
4421      Register reg = temp.AsRegister<Register>();
4422      if (current_method.IsRegister()) {
4423        method_reg = current_method.AsRegister<Register>();
4424      } else {
4425        DCHECK(invoke->GetLocations()->Intrinsified());
4426        DCHECK(!current_method.IsValid());
4427        method_reg = reg;
4428        __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
4429      }
4430      // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_;
4431      __ movl(reg, Address(method_reg,
4432                           ArtMethod::DexCacheResolvedMethodsOffset(kX86PointerSize).Int32Value()));
4433      // temp = temp[index_in_cache]
4434      uint32_t index_in_cache = invoke->GetTargetMethod().dex_method_index;
4435      __ movl(reg, Address(reg, CodeGenerator::GetCachePointerOffset(index_in_cache)));
4436      break;
4437    }
4438  }
4439
4440  switch (invoke->GetCodePtrLocation()) {
4441    case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf:
4442      __ call(GetFrameEntryLabel());
4443      break;
4444    case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: {
4445      relative_call_patches_.emplace_back(invoke->GetTargetMethod());
4446      Label* label = &relative_call_patches_.back().label;
4447      __ call(label);  // Bind to the patch label, override at link time.
4448      __ Bind(label);  // Bind the label at the end of the "call" insn.
4449      break;
4450    }
4451    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
4452    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
4453      // Filtered out by GetSupportedInvokeStaticOrDirectDispatch().
4454      LOG(FATAL) << "Unsupported";
4455      UNREACHABLE();
4456    case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
4457      // (callee_method + offset_of_quick_compiled_code)()
4458      __ call(Address(callee_method.AsRegister<Register>(),
4459                      ArtMethod::EntryPointFromQuickCompiledCodeOffset(
4460                          kX86WordSize).Int32Value()));
4461      break;
4462  }
4463
4464  DCHECK(!IsLeafMethod());
4465}
4466
4467void CodeGeneratorX86::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) {
4468  Register temp = temp_in.AsRegister<Register>();
4469  uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
4470      invoke->GetVTableIndex(), kX86PointerSize).Uint32Value();
4471
4472  // Use the calling convention instead of the location of the receiver, as
4473  // intrinsics may have put the receiver in a different register. In the intrinsics
4474  // slow path, the arguments have been moved to the right place, so here we are
4475  // guaranteed that the receiver is the first register of the calling convention.
4476  InvokeDexCallingConvention calling_convention;
4477  Register receiver = calling_convention.GetRegisterAt(0);
4478  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4479  // /* HeapReference<Class> */ temp = receiver->klass_
4480  __ movl(temp, Address(receiver, class_offset));
4481  MaybeRecordImplicitNullCheck(invoke);
4482  // Instead of simply (possibly) unpoisoning `temp` here, we should
4483  // emit a read barrier for the previous class reference load.
4484  // However this is not required in practice, as this is an
4485  // intermediate/temporary reference and because the current
4486  // concurrent copying collector keeps the from-space memory
4487  // intact/accessible until the end of the marking phase (the
4488  // concurrent copying collector may not in the future).
4489  __ MaybeUnpoisonHeapReference(temp);
4490  // temp = temp->GetMethodAt(method_offset);
4491  __ movl(temp, Address(temp, method_offset));
4492  // call temp->GetEntryPoint();
4493  __ call(Address(
4494      temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
4495}
4496
4497void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
4498  DCHECK(linker_patches->empty());
4499  size_t size =
4500      method_patches_.size() +
4501      relative_call_patches_.size() +
4502      pc_relative_dex_cache_patches_.size();
4503  linker_patches->reserve(size);
4504  // The label points to the end of the "movl" insn but the literal offset for method
4505  // patch needs to point to the embedded constant which occupies the last 4 bytes.
4506  constexpr uint32_t kLabelPositionToLiteralOffsetAdjustment = 4u;
4507  for (const MethodPatchInfo<Label>& info : method_patches_) {
4508    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4509    linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset,
4510                                                       info.target_method.dex_file,
4511                                                       info.target_method.dex_method_index));
4512  }
4513  for (const MethodPatchInfo<Label>& info : relative_call_patches_) {
4514    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4515    linker_patches->push_back(LinkerPatch::RelativeCodePatch(literal_offset,
4516                                                             info.target_method.dex_file,
4517                                                             info.target_method.dex_method_index));
4518  }
4519  for (const PcRelativeDexCacheAccessInfo& info : pc_relative_dex_cache_patches_) {
4520    uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4521    linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(literal_offset,
4522                                                              &info.target_dex_file,
4523                                                              GetMethodAddressOffset(),
4524                                                              info.element_offset));
4525  }
4526}
4527
4528void CodeGeneratorX86::MarkGCCard(Register temp,
4529                                  Register card,
4530                                  Register object,
4531                                  Register value,
4532                                  bool value_can_be_null) {
4533  NearLabel is_null;
4534  if (value_can_be_null) {
4535    __ testl(value, value);
4536    __ j(kEqual, &is_null);
4537  }
4538  __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
4539  __ movl(temp, object);
4540  __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
4541  __ movb(Address(temp, card, TIMES_1, 0),
4542          X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
4543  if (value_can_be_null) {
4544    __ Bind(&is_null);
4545  }
4546}
4547
4548void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
4549  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
4550
4551  bool object_field_get_with_read_barrier =
4552      kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
4553  LocationSummary* locations =
4554      new (GetGraph()->GetArena()) LocationSummary(instruction,
4555                                                   kEmitCompilerReadBarrier ?
4556                                                       LocationSummary::kCallOnSlowPath :
4557                                                       LocationSummary::kNoCall);
4558  locations->SetInAt(0, Location::RequiresRegister());
4559
4560  if (Primitive::IsFloatingPointType(instruction->GetType())) {
4561    locations->SetOut(Location::RequiresFpuRegister());
4562  } else {
4563    // The output overlaps in case of long: we don't want the low move
4564    // to overwrite the object's location.  Likewise, in the case of
4565    // an object field get with read barriers enabled, we do not want
4566    // the move to overwrite the object's location, as we need it to emit
4567    // the read barrier.
4568    locations->SetOut(
4569        Location::RequiresRegister(),
4570        (object_field_get_with_read_barrier || instruction->GetType() == Primitive::kPrimLong) ?
4571            Location::kOutputOverlap :
4572            Location::kNoOutputOverlap);
4573  }
4574
4575  if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
4576    // Long values can be loaded atomically into an XMM using movsd.
4577    // So we use an XMM register as a temp to achieve atomicity (first
4578    // load the temp into the XMM and then copy the XMM into the
4579    // output, 32 bits at a time).
4580    locations->AddTemp(Location::RequiresFpuRegister());
4581  } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) {
4582    // We need a temporary register for the read barrier marking slow
4583    // path in CodeGeneratorX86::GenerateFieldLoadWithBakerReadBarrier.
4584    locations->AddTemp(Location::RequiresRegister());
4585  }
4586}
4587
4588void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
4589                                                 const FieldInfo& field_info) {
4590  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
4591
4592  LocationSummary* locations = instruction->GetLocations();
4593  Location base_loc = locations->InAt(0);
4594  Register base = base_loc.AsRegister<Register>();
4595  Location out = locations->Out();
4596  bool is_volatile = field_info.IsVolatile();
4597  Primitive::Type field_type = field_info.GetFieldType();
4598  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
4599
4600  switch (field_type) {
4601    case Primitive::kPrimBoolean: {
4602      __ movzxb(out.AsRegister<Register>(), Address(base, offset));
4603      break;
4604    }
4605
4606    case Primitive::kPrimByte: {
4607      __ movsxb(out.AsRegister<Register>(), Address(base, offset));
4608      break;
4609    }
4610
4611    case Primitive::kPrimShort: {
4612      __ movsxw(out.AsRegister<Register>(), Address(base, offset));
4613      break;
4614    }
4615
4616    case Primitive::kPrimChar: {
4617      __ movzxw(out.AsRegister<Register>(), Address(base, offset));
4618      break;
4619    }
4620
4621    case Primitive::kPrimInt:
4622      __ movl(out.AsRegister<Register>(), Address(base, offset));
4623      break;
4624
4625    case Primitive::kPrimNot: {
4626      // /* HeapReference<Object> */ out = *(base + offset)
4627      if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
4628        Location temp_loc = locations->GetTemp(0);
4629        // Note that a potential implicit null check is handled in this
4630        // CodeGeneratorX86::GenerateFieldLoadWithBakerReadBarrier call.
4631        codegen_->GenerateFieldLoadWithBakerReadBarrier(
4632            instruction, out, base, offset, temp_loc, /* needs_null_check */ true);
4633        if (is_volatile) {
4634          codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4635        }
4636      } else {
4637        __ movl(out.AsRegister<Register>(), Address(base, offset));
4638        codegen_->MaybeRecordImplicitNullCheck(instruction);
4639        if (is_volatile) {
4640          codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4641        }
4642        // If read barriers are enabled, emit read barriers other than
4643        // Baker's using a slow path (and also unpoison the loaded
4644        // reference, if heap poisoning is enabled).
4645        codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset);
4646      }
4647      break;
4648    }
4649
4650    case Primitive::kPrimLong: {
4651      if (is_volatile) {
4652        XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
4653        __ movsd(temp, Address(base, offset));
4654        codegen_->MaybeRecordImplicitNullCheck(instruction);
4655        __ movd(out.AsRegisterPairLow<Register>(), temp);
4656        __ psrlq(temp, Immediate(32));
4657        __ movd(out.AsRegisterPairHigh<Register>(), temp);
4658      } else {
4659        DCHECK_NE(base, out.AsRegisterPairLow<Register>());
4660        __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
4661        codegen_->MaybeRecordImplicitNullCheck(instruction);
4662        __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
4663      }
4664      break;
4665    }
4666
4667    case Primitive::kPrimFloat: {
4668      __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
4669      break;
4670    }
4671
4672    case Primitive::kPrimDouble: {
4673      __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
4674      break;
4675    }
4676
4677    case Primitive::kPrimVoid:
4678      LOG(FATAL) << "Unreachable type " << field_type;
4679      UNREACHABLE();
4680  }
4681
4682  if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimLong) {
4683    // Potential implicit null checks, in the case of reference or
4684    // long fields, are handled in the previous switch statement.
4685  } else {
4686    codegen_->MaybeRecordImplicitNullCheck(instruction);
4687  }
4688
4689  if (is_volatile) {
4690    if (field_type == Primitive::kPrimNot) {
4691      // Memory barriers, in the case of references, are also handled
4692      // in the previous switch statement.
4693    } else {
4694      codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4695    }
4696  }
4697}
4698
4699void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
4700  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
4701
4702  LocationSummary* locations =
4703      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4704  locations->SetInAt(0, Location::RequiresRegister());
4705  bool is_volatile = field_info.IsVolatile();
4706  Primitive::Type field_type = field_info.GetFieldType();
4707  bool is_byte_type = (field_type == Primitive::kPrimBoolean)
4708    || (field_type == Primitive::kPrimByte);
4709
4710  // The register allocator does not support multiple
4711  // inputs that die at entry with one in a specific register.
4712  if (is_byte_type) {
4713    // Ensure the value is in a byte register.
4714    locations->SetInAt(1, Location::RegisterLocation(EAX));
4715  } else if (Primitive::IsFloatingPointType(field_type)) {
4716    if (is_volatile && field_type == Primitive::kPrimDouble) {
4717      // In order to satisfy the semantics of volatile, this must be a single instruction store.
4718      locations->SetInAt(1, Location::RequiresFpuRegister());
4719    } else {
4720      locations->SetInAt(1, Location::FpuRegisterOrConstant(instruction->InputAt(1)));
4721    }
4722  } else if (is_volatile && field_type == Primitive::kPrimLong) {
4723    // In order to satisfy the semantics of volatile, this must be a single instruction store.
4724    locations->SetInAt(1, Location::RequiresRegister());
4725
4726    // 64bits value can be atomically written to an address with movsd and an XMM register.
4727    // We need two XMM registers because there's no easier way to (bit) copy a register pair
4728    // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
4729    // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
4730    // isolated cases when we need this it isn't worth adding the extra complexity.
4731    locations->AddTemp(Location::RequiresFpuRegister());
4732    locations->AddTemp(Location::RequiresFpuRegister());
4733  } else {
4734    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4735
4736    if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
4737      // Temporary registers for the write barrier.
4738      locations->AddTemp(Location::RequiresRegister());  // May be used for reference poisoning too.
4739      // Ensure the card is in a byte register.
4740      locations->AddTemp(Location::RegisterLocation(ECX));
4741    }
4742  }
4743}
4744
4745void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
4746                                                 const FieldInfo& field_info,
4747                                                 bool value_can_be_null) {
4748  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
4749
4750  LocationSummary* locations = instruction->GetLocations();
4751  Register base = locations->InAt(0).AsRegister<Register>();
4752  Location value = locations->InAt(1);
4753  bool is_volatile = field_info.IsVolatile();
4754  Primitive::Type field_type = field_info.GetFieldType();
4755  uint32_t offset = field_info.GetFieldOffset().Uint32Value();
4756  bool needs_write_barrier =
4757      CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1));
4758
4759  if (is_volatile) {
4760    codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
4761  }
4762
4763  bool maybe_record_implicit_null_check_done = false;
4764
4765  switch (field_type) {
4766    case Primitive::kPrimBoolean:
4767    case Primitive::kPrimByte: {
4768      __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
4769      break;
4770    }
4771
4772    case Primitive::kPrimShort:
4773    case Primitive::kPrimChar: {
4774      if (value.IsConstant()) {
4775        int16_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4776        __ movw(Address(base, offset), Immediate(v));
4777      } else {
4778        __ movw(Address(base, offset), value.AsRegister<Register>());
4779      }
4780      break;
4781    }
4782
4783    case Primitive::kPrimInt:
4784    case Primitive::kPrimNot: {
4785      if (kPoisonHeapReferences && needs_write_barrier) {
4786        // Note that in the case where `value` is a null reference,
4787        // we do not enter this block, as the reference does not
4788        // need poisoning.
4789        DCHECK_EQ(field_type, Primitive::kPrimNot);
4790        Register temp = locations->GetTemp(0).AsRegister<Register>();
4791        __ movl(temp, value.AsRegister<Register>());
4792        __ PoisonHeapReference(temp);
4793        __ movl(Address(base, offset), temp);
4794      } else if (value.IsConstant()) {
4795        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4796        __ movl(Address(base, offset), Immediate(v));
4797      } else {
4798        __ movl(Address(base, offset), value.AsRegister<Register>());
4799      }
4800      break;
4801    }
4802
4803    case Primitive::kPrimLong: {
4804      if (is_volatile) {
4805        XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
4806        XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
4807        __ movd(temp1, value.AsRegisterPairLow<Register>());
4808        __ movd(temp2, value.AsRegisterPairHigh<Register>());
4809        __ punpckldq(temp1, temp2);
4810        __ movsd(Address(base, offset), temp1);
4811        codegen_->MaybeRecordImplicitNullCheck(instruction);
4812      } else if (value.IsConstant()) {
4813        int64_t v = CodeGenerator::GetInt64ValueOf(value.GetConstant());
4814        __ movl(Address(base, offset), Immediate(Low32Bits(v)));
4815        codegen_->MaybeRecordImplicitNullCheck(instruction);
4816        __ movl(Address(base, kX86WordSize + offset), Immediate(High32Bits(v)));
4817      } else {
4818        __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
4819        codegen_->MaybeRecordImplicitNullCheck(instruction);
4820        __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
4821      }
4822      maybe_record_implicit_null_check_done = true;
4823      break;
4824    }
4825
4826    case Primitive::kPrimFloat: {
4827      if (value.IsConstant()) {
4828        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4829        __ movl(Address(base, offset), Immediate(v));
4830      } else {
4831        __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
4832      }
4833      break;
4834    }
4835
4836    case Primitive::kPrimDouble: {
4837      if (value.IsConstant()) {
4838        int64_t v = CodeGenerator::GetInt64ValueOf(value.GetConstant());
4839        __ movl(Address(base, offset), Immediate(Low32Bits(v)));
4840        codegen_->MaybeRecordImplicitNullCheck(instruction);
4841        __ movl(Address(base, kX86WordSize + offset), Immediate(High32Bits(v)));
4842        maybe_record_implicit_null_check_done = true;
4843      } else {
4844        __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
4845      }
4846      break;
4847    }
4848
4849    case Primitive::kPrimVoid:
4850      LOG(FATAL) << "Unreachable type " << field_type;
4851      UNREACHABLE();
4852  }
4853
4854  if (!maybe_record_implicit_null_check_done) {
4855    codegen_->MaybeRecordImplicitNullCheck(instruction);
4856  }
4857
4858  if (needs_write_barrier) {
4859    Register temp = locations->GetTemp(0).AsRegister<Register>();
4860    Register card = locations->GetTemp(1).AsRegister<Register>();
4861    codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>(), value_can_be_null);
4862  }
4863
4864  if (is_volatile) {
4865    codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
4866  }
4867}
4868
4869void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4870  HandleFieldGet(instruction, instruction->GetFieldInfo());
4871}
4872
4873void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4874  HandleFieldGet(instruction, instruction->GetFieldInfo());
4875}
4876
4877void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4878  HandleFieldSet(instruction, instruction->GetFieldInfo());
4879}
4880
4881void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4882  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4883}
4884
4885void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4886  HandleFieldSet(instruction, instruction->GetFieldInfo());
4887}
4888
4889void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4890  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4891}
4892
4893void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4894  HandleFieldGet(instruction, instruction->GetFieldInfo());
4895}
4896
4897void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4898  HandleFieldGet(instruction, instruction->GetFieldInfo());
4899}
4900
4901void LocationsBuilderX86::VisitUnresolvedInstanceFieldGet(
4902    HUnresolvedInstanceFieldGet* instruction) {
4903  FieldAccessCallingConventionX86 calling_convention;
4904  codegen_->CreateUnresolvedFieldLocationSummary(
4905      instruction, instruction->GetFieldType(), calling_convention);
4906}
4907
4908void InstructionCodeGeneratorX86::VisitUnresolvedInstanceFieldGet(
4909    HUnresolvedInstanceFieldGet* instruction) {
4910  FieldAccessCallingConventionX86 calling_convention;
4911  codegen_->GenerateUnresolvedFieldAccess(instruction,
4912                                          instruction->GetFieldType(),
4913                                          instruction->GetFieldIndex(),
4914                                          instruction->GetDexPc(),
4915                                          calling_convention);
4916}
4917
4918void LocationsBuilderX86::VisitUnresolvedInstanceFieldSet(
4919    HUnresolvedInstanceFieldSet* instruction) {
4920  FieldAccessCallingConventionX86 calling_convention;
4921  codegen_->CreateUnresolvedFieldLocationSummary(
4922      instruction, instruction->GetFieldType(), calling_convention);
4923}
4924
4925void InstructionCodeGeneratorX86::VisitUnresolvedInstanceFieldSet(
4926    HUnresolvedInstanceFieldSet* instruction) {
4927  FieldAccessCallingConventionX86 calling_convention;
4928  codegen_->GenerateUnresolvedFieldAccess(instruction,
4929                                          instruction->GetFieldType(),
4930                                          instruction->GetFieldIndex(),
4931                                          instruction->GetDexPc(),
4932                                          calling_convention);
4933}
4934
4935void LocationsBuilderX86::VisitUnresolvedStaticFieldGet(
4936    HUnresolvedStaticFieldGet* instruction) {
4937  FieldAccessCallingConventionX86 calling_convention;
4938  codegen_->CreateUnresolvedFieldLocationSummary(
4939      instruction, instruction->GetFieldType(), calling_convention);
4940}
4941
4942void InstructionCodeGeneratorX86::VisitUnresolvedStaticFieldGet(
4943    HUnresolvedStaticFieldGet* instruction) {
4944  FieldAccessCallingConventionX86 calling_convention;
4945  codegen_->GenerateUnresolvedFieldAccess(instruction,
4946                                          instruction->GetFieldType(),
4947                                          instruction->GetFieldIndex(),
4948                                          instruction->GetDexPc(),
4949                                          calling_convention);
4950}
4951
4952void LocationsBuilderX86::VisitUnresolvedStaticFieldSet(
4953    HUnresolvedStaticFieldSet* instruction) {
4954  FieldAccessCallingConventionX86 calling_convention;
4955  codegen_->CreateUnresolvedFieldLocationSummary(
4956      instruction, instruction->GetFieldType(), calling_convention);
4957}
4958
4959void InstructionCodeGeneratorX86::VisitUnresolvedStaticFieldSet(
4960    HUnresolvedStaticFieldSet* instruction) {
4961  FieldAccessCallingConventionX86 calling_convention;
4962  codegen_->GenerateUnresolvedFieldAccess(instruction,
4963                                          instruction->GetFieldType(),
4964                                          instruction->GetFieldIndex(),
4965                                          instruction->GetDexPc(),
4966                                          calling_convention);
4967}
4968
4969void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
4970  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
4971      ? LocationSummary::kCallOnSlowPath
4972      : LocationSummary::kNoCall;
4973  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4974  Location loc = codegen_->IsImplicitNullCheckAllowed(instruction)
4975      ? Location::RequiresRegister()
4976      : Location::Any();
4977  locations->SetInAt(0, loc);
4978  if (instruction->HasUses()) {
4979    locations->SetOut(Location::SameAsFirstInput());
4980  }
4981}
4982
4983void CodeGeneratorX86::GenerateImplicitNullCheck(HNullCheck* instruction) {
4984  if (CanMoveNullCheckToUser(instruction)) {
4985    return;
4986  }
4987  LocationSummary* locations = instruction->GetLocations();
4988  Location obj = locations->InAt(0);
4989
4990  __ testl(EAX, Address(obj.AsRegister<Register>(), 0));
4991  RecordPcInfo(instruction, instruction->GetDexPc());
4992}
4993
4994void CodeGeneratorX86::GenerateExplicitNullCheck(HNullCheck* instruction) {
4995  SlowPathCode* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
4996  AddSlowPath(slow_path);
4997
4998  LocationSummary* locations = instruction->GetLocations();
4999  Location obj = locations->InAt(0);
5000
5001  if (obj.IsRegister()) {
5002    __ testl(obj.AsRegister<Register>(), obj.AsRegister<Register>());
5003  } else if (obj.IsStackSlot()) {
5004    __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
5005  } else {
5006    DCHECK(obj.IsConstant()) << obj;
5007    DCHECK(obj.GetConstant()->IsNullConstant());
5008    __ jmp(slow_path->GetEntryLabel());
5009    return;
5010  }
5011  __ j(kEqual, slow_path->GetEntryLabel());
5012}
5013
5014void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
5015  codegen_->GenerateNullCheck(instruction);
5016}
5017
5018void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
5019  bool object_array_get_with_read_barrier =
5020      kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
5021  LocationSummary* locations =
5022      new (GetGraph()->GetArena()) LocationSummary(instruction,
5023                                                   object_array_get_with_read_barrier ?
5024                                                       LocationSummary::kCallOnSlowPath :
5025                                                       LocationSummary::kNoCall);
5026  locations->SetInAt(0, Location::RequiresRegister());
5027  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
5028  if (Primitive::IsFloatingPointType(instruction->GetType())) {
5029    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
5030  } else {
5031    // The output overlaps in case of long: we don't want the low move
5032    // to overwrite the array's location.  Likewise, in the case of an
5033    // object array get with read barriers enabled, we do not want the
5034    // move to overwrite the array's location, as we need it to emit
5035    // the read barrier.
5036    locations->SetOut(
5037        Location::RequiresRegister(),
5038        (instruction->GetType() == Primitive::kPrimLong || object_array_get_with_read_barrier) ?
5039            Location::kOutputOverlap :
5040            Location::kNoOutputOverlap);
5041  }
5042  // We need a temporary register for the read barrier marking slow
5043  // path in CodeGeneratorX86::GenerateArrayLoadWithBakerReadBarrier.
5044  if (object_array_get_with_read_barrier && kUseBakerReadBarrier) {
5045    locations->AddTemp(Location::RequiresRegister());
5046  }
5047}
5048
5049void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
5050  LocationSummary* locations = instruction->GetLocations();
5051  Location obj_loc = locations->InAt(0);
5052  Register obj = obj_loc.AsRegister<Register>();
5053  Location index = locations->InAt(1);
5054  Location out_loc = locations->Out();
5055
5056  Primitive::Type type = instruction->GetType();
5057  switch (type) {
5058    case Primitive::kPrimBoolean: {
5059      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
5060      Register out = out_loc.AsRegister<Register>();
5061      if (index.IsConstant()) {
5062        __ movzxb(out, Address(obj,
5063            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
5064      } else {
5065        __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
5066      }
5067      break;
5068    }
5069
5070    case Primitive::kPrimByte: {
5071      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
5072      Register out = out_loc.AsRegister<Register>();
5073      if (index.IsConstant()) {
5074        __ movsxb(out, Address(obj,
5075            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
5076      } else {
5077        __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
5078      }
5079      break;
5080    }
5081
5082    case Primitive::kPrimShort: {
5083      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
5084      Register out = out_loc.AsRegister<Register>();
5085      if (index.IsConstant()) {
5086        __ movsxw(out, Address(obj,
5087            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
5088      } else {
5089        __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
5090      }
5091      break;
5092    }
5093
5094    case Primitive::kPrimChar: {
5095      uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
5096      Register out = out_loc.AsRegister<Register>();
5097      if (index.IsConstant()) {
5098        __ movzxw(out, Address(obj,
5099            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
5100      } else {
5101        __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
5102      }
5103      break;
5104    }
5105
5106    case Primitive::kPrimInt: {
5107      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
5108      Register out = out_loc.AsRegister<Register>();
5109      if (index.IsConstant()) {
5110        __ movl(out, Address(obj,
5111            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
5112      } else {
5113        __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
5114      }
5115      break;
5116    }
5117
5118    case Primitive::kPrimNot: {
5119      static_assert(
5120          sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
5121          "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
5122      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
5123      // /* HeapReference<Object> */ out =
5124      //     *(obj + data_offset + index * sizeof(HeapReference<Object>))
5125      if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
5126        Location temp = locations->GetTemp(0);
5127        // Note that a potential implicit null check is handled in this
5128        // CodeGeneratorX86::GenerateArrayLoadWithBakerReadBarrier call.
5129        codegen_->GenerateArrayLoadWithBakerReadBarrier(
5130            instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true);
5131      } else {
5132        Register out = out_loc.AsRegister<Register>();
5133        if (index.IsConstant()) {
5134          uint32_t offset =
5135              (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
5136          __ movl(out, Address(obj, offset));
5137          codegen_->MaybeRecordImplicitNullCheck(instruction);
5138          // If read barriers are enabled, emit read barriers other than
5139          // Baker's using a slow path (and also unpoison the loaded
5140          // reference, if heap poisoning is enabled).
5141          codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset);
5142        } else {
5143          __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
5144          codegen_->MaybeRecordImplicitNullCheck(instruction);
5145          // If read barriers are enabled, emit read barriers other than
5146          // Baker's using a slow path (and also unpoison the loaded
5147          // reference, if heap poisoning is enabled).
5148          codegen_->MaybeGenerateReadBarrierSlow(
5149              instruction, out_loc, out_loc, obj_loc, data_offset, index);
5150        }
5151      }
5152      break;
5153    }
5154
5155    case Primitive::kPrimLong: {
5156      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
5157      DCHECK_NE(obj, out_loc.AsRegisterPairLow<Register>());
5158      if (index.IsConstant()) {
5159        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
5160        __ movl(out_loc.AsRegisterPairLow<Register>(), Address(obj, offset));
5161        codegen_->MaybeRecordImplicitNullCheck(instruction);
5162        __ movl(out_loc.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
5163      } else {
5164        __ movl(out_loc.AsRegisterPairLow<Register>(),
5165                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
5166        codegen_->MaybeRecordImplicitNullCheck(instruction);
5167        __ movl(out_loc.AsRegisterPairHigh<Register>(),
5168                Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
5169      }
5170      break;
5171    }
5172
5173    case Primitive::kPrimFloat: {
5174      uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
5175      XmmRegister out = out_loc.AsFpuRegister<XmmRegister>();
5176      if (index.IsConstant()) {
5177        __ movss(out, Address(obj,
5178            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
5179      } else {
5180        __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
5181      }
5182      break;
5183    }
5184
5185    case Primitive::kPrimDouble: {
5186      uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
5187      XmmRegister out = out_loc.AsFpuRegister<XmmRegister>();
5188      if (index.IsConstant()) {
5189        __ movsd(out, Address(obj,
5190            (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
5191      } else {
5192        __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
5193      }
5194      break;
5195    }
5196
5197    case Primitive::kPrimVoid:
5198      LOG(FATAL) << "Unreachable type " << type;
5199      UNREACHABLE();
5200  }
5201
5202  if (type == Primitive::kPrimNot || type == Primitive::kPrimLong) {
5203    // Potential implicit null checks, in the case of reference or
5204    // long arrays, are handled in the previous switch statement.
5205  } else {
5206    codegen_->MaybeRecordImplicitNullCheck(instruction);
5207  }
5208}
5209
5210void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
5211  Primitive::Type value_type = instruction->GetComponentType();
5212
5213  bool needs_write_barrier =
5214      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
5215  bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
5216  bool object_array_set_with_read_barrier =
5217      kEmitCompilerReadBarrier && (value_type == Primitive::kPrimNot);
5218
5219  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
5220      instruction,
5221      (may_need_runtime_call_for_type_check || object_array_set_with_read_barrier) ?
5222          LocationSummary::kCallOnSlowPath :
5223          LocationSummary::kNoCall);
5224
5225  bool is_byte_type = (value_type == Primitive::kPrimBoolean)
5226      || (value_type == Primitive::kPrimByte);
5227  // We need the inputs to be different than the output in case of long operation.
5228  // In case of a byte operation, the register allocator does not support multiple
5229  // inputs that die at entry with one in a specific register.
5230  locations->SetInAt(0, Location::RequiresRegister());
5231  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
5232  if (is_byte_type) {
5233    // Ensure the value is in a byte register.
5234    locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
5235  } else if (Primitive::IsFloatingPointType(value_type)) {
5236    locations->SetInAt(2, Location::FpuRegisterOrConstant(instruction->InputAt(2)));
5237  } else {
5238    locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
5239  }
5240  if (needs_write_barrier) {
5241    // Temporary registers for the write barrier.
5242    locations->AddTemp(Location::RequiresRegister());  // Possibly used for ref. poisoning too.
5243    // Ensure the card is in a byte register.
5244    locations->AddTemp(Location::RegisterLocation(ECX));
5245  }
5246}
5247
5248void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
5249  LocationSummary* locations = instruction->GetLocations();
5250  Location array_loc = locations->InAt(0);
5251  Register array = array_loc.AsRegister<Register>();
5252  Location index = locations->InAt(1);
5253  Location value = locations->InAt(2);
5254  Primitive::Type value_type = instruction->GetComponentType();
5255  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5256  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
5257  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
5258  bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
5259  bool needs_write_barrier =
5260      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
5261
5262  switch (value_type) {
5263    case Primitive::kPrimBoolean:
5264    case Primitive::kPrimByte: {
5265      uint32_t offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
5266      Address address = index.IsConstant()
5267          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + offset)
5268          : Address(array, index.AsRegister<Register>(), TIMES_1, offset);
5269      if (value.IsRegister()) {
5270        __ movb(address, value.AsRegister<ByteRegister>());
5271      } else {
5272        __ movb(address, Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
5273      }
5274      codegen_->MaybeRecordImplicitNullCheck(instruction);
5275      break;
5276    }
5277
5278    case Primitive::kPrimShort:
5279    case Primitive::kPrimChar: {
5280      uint32_t offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
5281      Address address = index.IsConstant()
5282          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + offset)
5283          : Address(array, index.AsRegister<Register>(), TIMES_2, offset);
5284      if (value.IsRegister()) {
5285        __ movw(address, value.AsRegister<Register>());
5286      } else {
5287        __ movw(address, Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
5288      }
5289      codegen_->MaybeRecordImplicitNullCheck(instruction);
5290      break;
5291    }
5292
5293    case Primitive::kPrimNot: {
5294      uint32_t offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
5295      Address address = index.IsConstant()
5296          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5297          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5298
5299      if (!value.IsRegister()) {
5300        // Just setting null.
5301        DCHECK(instruction->InputAt(2)->IsNullConstant());
5302        DCHECK(value.IsConstant()) << value;
5303        __ movl(address, Immediate(0));
5304        codegen_->MaybeRecordImplicitNullCheck(instruction);
5305        DCHECK(!needs_write_barrier);
5306        DCHECK(!may_need_runtime_call_for_type_check);
5307        break;
5308      }
5309
5310      DCHECK(needs_write_barrier);
5311      Register register_value = value.AsRegister<Register>();
5312      NearLabel done, not_null, do_put;
5313      SlowPathCode* slow_path = nullptr;
5314      Register temp = locations->GetTemp(0).AsRegister<Register>();
5315      if (may_need_runtime_call_for_type_check) {
5316        slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathX86(instruction);
5317        codegen_->AddSlowPath(slow_path);
5318        if (instruction->GetValueCanBeNull()) {
5319          __ testl(register_value, register_value);
5320          __ j(kNotEqual, &not_null);
5321          __ movl(address, Immediate(0));
5322          codegen_->MaybeRecordImplicitNullCheck(instruction);
5323          __ jmp(&done);
5324          __ Bind(&not_null);
5325        }
5326
5327        if (kEmitCompilerReadBarrier) {
5328          // When read barriers are enabled, the type checking
5329          // instrumentation requires two read barriers:
5330          //
5331          //   __ movl(temp2, temp);
5332          //   // /* HeapReference<Class> */ temp = temp->component_type_
5333          //   __ movl(temp, Address(temp, component_offset));
5334          //   codegen_->GenerateReadBarrierSlow(
5335          //       instruction, temp_loc, temp_loc, temp2_loc, component_offset);
5336          //
5337          //   // /* HeapReference<Class> */ temp2 = register_value->klass_
5338          //   __ movl(temp2, Address(register_value, class_offset));
5339          //   codegen_->GenerateReadBarrierSlow(
5340          //       instruction, temp2_loc, temp2_loc, value, class_offset, temp_loc);
5341          //
5342          //   __ cmpl(temp, temp2);
5343          //
5344          // However, the second read barrier may trash `temp`, as it
5345          // is a temporary register, and as such would not be saved
5346          // along with live registers before calling the runtime (nor
5347          // restored afterwards).  So in this case, we bail out and
5348          // delegate the work to the array set slow path.
5349          //
5350          // TODO: Extend the register allocator to support a new
5351          // "(locally) live temp" location so as to avoid always
5352          // going into the slow path when read barriers are enabled.
5353          __ jmp(slow_path->GetEntryLabel());
5354        } else {
5355          // /* HeapReference<Class> */ temp = array->klass_
5356          __ movl(temp, Address(array, class_offset));
5357          codegen_->MaybeRecordImplicitNullCheck(instruction);
5358          __ MaybeUnpoisonHeapReference(temp);
5359
5360          // /* HeapReference<Class> */ temp = temp->component_type_
5361          __ movl(temp, Address(temp, component_offset));
5362          // If heap poisoning is enabled, no need to unpoison `temp`
5363          // nor the object reference in `register_value->klass`, as
5364          // we are comparing two poisoned references.
5365          __ cmpl(temp, Address(register_value, class_offset));
5366
5367          if (instruction->StaticTypeOfArrayIsObjectArray()) {
5368            __ j(kEqual, &do_put);
5369            // If heap poisoning is enabled, the `temp` reference has
5370            // not been unpoisoned yet; unpoison it now.
5371            __ MaybeUnpoisonHeapReference(temp);
5372
5373            // /* HeapReference<Class> */ temp = temp->super_class_
5374            __ movl(temp, Address(temp, super_offset));
5375            // If heap poisoning is enabled, no need to unpoison
5376            // `temp`, as we are comparing against null below.
5377            __ testl(temp, temp);
5378            __ j(kNotEqual, slow_path->GetEntryLabel());
5379            __ Bind(&do_put);
5380          } else {
5381            __ j(kNotEqual, slow_path->GetEntryLabel());
5382          }
5383        }
5384      }
5385
5386      if (kPoisonHeapReferences) {
5387        __ movl(temp, register_value);
5388        __ PoisonHeapReference(temp);
5389        __ movl(address, temp);
5390      } else {
5391        __ movl(address, register_value);
5392      }
5393      if (!may_need_runtime_call_for_type_check) {
5394        codegen_->MaybeRecordImplicitNullCheck(instruction);
5395      }
5396
5397      Register card = locations->GetTemp(1).AsRegister<Register>();
5398      codegen_->MarkGCCard(
5399          temp, card, array, value.AsRegister<Register>(), instruction->GetValueCanBeNull());
5400      __ Bind(&done);
5401
5402      if (slow_path != nullptr) {
5403        __ Bind(slow_path->GetExitLabel());
5404      }
5405
5406      break;
5407    }
5408
5409    case Primitive::kPrimInt: {
5410      uint32_t offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
5411      Address address = index.IsConstant()
5412          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5413          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5414      if (value.IsRegister()) {
5415        __ movl(address, value.AsRegister<Register>());
5416      } else {
5417        DCHECK(value.IsConstant()) << value;
5418        int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
5419        __ movl(address, Immediate(v));
5420      }
5421      codegen_->MaybeRecordImplicitNullCheck(instruction);
5422      break;
5423    }
5424
5425    case Primitive::kPrimLong: {
5426      uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
5427      if (index.IsConstant()) {
5428        size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
5429        if (value.IsRegisterPair()) {
5430          __ movl(Address(array, offset), value.AsRegisterPairLow<Register>());
5431          codegen_->MaybeRecordImplicitNullCheck(instruction);
5432          __ movl(Address(array, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
5433        } else {
5434          DCHECK(value.IsConstant());
5435          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
5436          __ movl(Address(array, offset), Immediate(Low32Bits(val)));
5437          codegen_->MaybeRecordImplicitNullCheck(instruction);
5438          __ movl(Address(array, offset + kX86WordSize), Immediate(High32Bits(val)));
5439        }
5440      } else {
5441        if (value.IsRegisterPair()) {
5442          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset),
5443                  value.AsRegisterPairLow<Register>());
5444          codegen_->MaybeRecordImplicitNullCheck(instruction);
5445          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
5446                  value.AsRegisterPairHigh<Register>());
5447        } else {
5448          DCHECK(value.IsConstant());
5449          int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
5450          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset),
5451                  Immediate(Low32Bits(val)));
5452          codegen_->MaybeRecordImplicitNullCheck(instruction);
5453          __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
5454                  Immediate(High32Bits(val)));
5455        }
5456      }
5457      break;
5458    }
5459
5460    case Primitive::kPrimFloat: {
5461      uint32_t offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
5462      Address address = index.IsConstant()
5463          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5464          : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5465      if (value.IsFpuRegister()) {
5466        __ movss(address, value.AsFpuRegister<XmmRegister>());
5467      } else {
5468        DCHECK(value.IsConstant());
5469        int32_t v = bit_cast<int32_t, float>(value.GetConstant()->AsFloatConstant()->GetValue());
5470        __ movl(address, Immediate(v));
5471      }
5472      codegen_->MaybeRecordImplicitNullCheck(instruction);
5473      break;
5474    }
5475
5476    case Primitive::kPrimDouble: {
5477      uint32_t offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
5478      Address address = index.IsConstant()
5479          ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + offset)
5480          : Address(array, index.AsRegister<Register>(), TIMES_8, offset);
5481      if (value.IsFpuRegister()) {
5482        __ movsd(address, value.AsFpuRegister<XmmRegister>());
5483      } else {
5484        DCHECK(value.IsConstant());
5485        Address address_hi = index.IsConstant() ?
5486            Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) +
5487                           offset + kX86WordSize) :
5488            Address(array, index.AsRegister<Register>(), TIMES_8, offset + kX86WordSize);
5489        int64_t v = bit_cast<int64_t, double>(value.GetConstant()->AsDoubleConstant()->GetValue());
5490        __ movl(address, Immediate(Low32Bits(v)));
5491        codegen_->MaybeRecordImplicitNullCheck(instruction);
5492        __ movl(address_hi, Immediate(High32Bits(v)));
5493      }
5494      break;
5495    }
5496
5497    case Primitive::kPrimVoid:
5498      LOG(FATAL) << "Unreachable type " << instruction->GetType();
5499      UNREACHABLE();
5500  }
5501}
5502
5503void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
5504  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
5505  locations->SetInAt(0, Location::RequiresRegister());
5506  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
5507}
5508
5509void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
5510  LocationSummary* locations = instruction->GetLocations();
5511  uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
5512  Register obj = locations->InAt(0).AsRegister<Register>();
5513  Register out = locations->Out().AsRegister<Register>();
5514  __ movl(out, Address(obj, offset));
5515  codegen_->MaybeRecordImplicitNullCheck(instruction);
5516}
5517
5518void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
5519  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
5520      ? LocationSummary::kCallOnSlowPath
5521      : LocationSummary::kNoCall;
5522  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5523  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
5524  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
5525  if (instruction->HasUses()) {
5526    locations->SetOut(Location::SameAsFirstInput());
5527  }
5528}
5529
5530void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
5531  LocationSummary* locations = instruction->GetLocations();
5532  Location index_loc = locations->InAt(0);
5533  Location length_loc = locations->InAt(1);
5534  SlowPathCode* slow_path =
5535    new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(instruction);
5536
5537  if (length_loc.IsConstant()) {
5538    int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
5539    if (index_loc.IsConstant()) {
5540      // BCE will remove the bounds check if we are guarenteed to pass.
5541      int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
5542      if (index < 0 || index >= length) {
5543        codegen_->AddSlowPath(slow_path);
5544        __ jmp(slow_path->GetEntryLabel());
5545      } else {
5546        // Some optimization after BCE may have generated this, and we should not
5547        // generate a bounds check if it is a valid range.
5548      }
5549      return;
5550    }
5551
5552    // We have to reverse the jump condition because the length is the constant.
5553    Register index_reg = index_loc.AsRegister<Register>();
5554    __ cmpl(index_reg, Immediate(length));
5555    codegen_->AddSlowPath(slow_path);
5556    __ j(kAboveEqual, slow_path->GetEntryLabel());
5557  } else {
5558    Register length = length_loc.AsRegister<Register>();
5559    if (index_loc.IsConstant()) {
5560      int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
5561      __ cmpl(length, Immediate(value));
5562    } else {
5563      __ cmpl(length, index_loc.AsRegister<Register>());
5564    }
5565    codegen_->AddSlowPath(slow_path);
5566    __ j(kBelowEqual, slow_path->GetEntryLabel());
5567  }
5568}
5569
5570void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) {
5571  LOG(FATAL) << "Unreachable";
5572}
5573
5574void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
5575  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
5576}
5577
5578void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
5579  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
5580}
5581
5582void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
5583  HBasicBlock* block = instruction->GetBlock();
5584  if (block->GetLoopInformation() != nullptr) {
5585    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
5586    // The back edge will generate the suspend check.
5587    return;
5588  }
5589  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
5590    // The goto will generate the suspend check.
5591    return;
5592  }
5593  GenerateSuspendCheck(instruction, nullptr);
5594}
5595
5596void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
5597                                                       HBasicBlock* successor) {
5598  SuspendCheckSlowPathX86* slow_path =
5599      down_cast<SuspendCheckSlowPathX86*>(instruction->GetSlowPath());
5600  if (slow_path == nullptr) {
5601    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
5602    instruction->SetSlowPath(slow_path);
5603    codegen_->AddSlowPath(slow_path);
5604    if (successor != nullptr) {
5605      DCHECK(successor->IsLoopHeader());
5606      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
5607    }
5608  } else {
5609    DCHECK_EQ(slow_path->GetSuccessor(), successor);
5610  }
5611
5612  __ fs()->cmpw(Address::Absolute(Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()),
5613                Immediate(0));
5614  if (successor == nullptr) {
5615    __ j(kNotEqual, slow_path->GetEntryLabel());
5616    __ Bind(slow_path->GetReturnLabel());
5617  } else {
5618    __ j(kEqual, codegen_->GetLabelOf(successor));
5619    __ jmp(slow_path->GetEntryLabel());
5620  }
5621}
5622
5623X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
5624  return codegen_->GetAssembler();
5625}
5626
5627void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) {
5628  ScratchRegisterScope ensure_scratch(
5629      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5630  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5631  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5632  __ movl(temp_reg, Address(ESP, src + stack_offset));
5633  __ movl(Address(ESP, dst + stack_offset), temp_reg);
5634}
5635
5636void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) {
5637  ScratchRegisterScope ensure_scratch(
5638      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5639  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5640  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5641  __ movl(temp_reg, Address(ESP, src + stack_offset));
5642  __ movl(Address(ESP, dst + stack_offset), temp_reg);
5643  __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize));
5644  __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg);
5645}
5646
5647void ParallelMoveResolverX86::EmitMove(size_t index) {
5648  MoveOperands* move = moves_[index];
5649  Location source = move->GetSource();
5650  Location destination = move->GetDestination();
5651
5652  if (source.IsRegister()) {
5653    if (destination.IsRegister()) {
5654      __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5655    } else if (destination.IsFpuRegister()) {
5656      __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
5657    } else {
5658      DCHECK(destination.IsStackSlot());
5659      __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
5660    }
5661  } else if (source.IsRegisterPair()) {
5662      size_t elem_size = Primitive::ComponentSize(Primitive::kPrimInt);
5663      // Create stack space for 2 elements.
5664      __ subl(ESP, Immediate(2 * elem_size));
5665      __ movl(Address(ESP, 0), source.AsRegisterPairLow<Register>());
5666      __ movl(Address(ESP, elem_size), source.AsRegisterPairHigh<Register>());
5667      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
5668      // And remove the temporary stack space we allocated.
5669      __ addl(ESP, Immediate(2 * elem_size));
5670  } else if (source.IsFpuRegister()) {
5671    if (destination.IsRegister()) {
5672      __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
5673    } else if (destination.IsFpuRegister()) {
5674      __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5675    } else if (destination.IsRegisterPair()) {
5676      XmmRegister src_reg = source.AsFpuRegister<XmmRegister>();
5677      __ movd(destination.AsRegisterPairLow<Register>(), src_reg);
5678      __ psrlq(src_reg, Immediate(32));
5679      __ movd(destination.AsRegisterPairHigh<Register>(), src_reg);
5680    } else if (destination.IsStackSlot()) {
5681      __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
5682    } else {
5683      DCHECK(destination.IsDoubleStackSlot());
5684      __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
5685    }
5686  } else if (source.IsStackSlot()) {
5687    if (destination.IsRegister()) {
5688      __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
5689    } else if (destination.IsFpuRegister()) {
5690      __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
5691    } else {
5692      DCHECK(destination.IsStackSlot());
5693      MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex());
5694    }
5695  } else if (source.IsDoubleStackSlot()) {
5696    if (destination.IsRegisterPair()) {
5697      __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
5698      __ movl(destination.AsRegisterPairHigh<Register>(),
5699              Address(ESP, source.GetHighStackIndex(kX86WordSize)));
5700    } else if (destination.IsFpuRegister()) {
5701      __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
5702    } else {
5703      DCHECK(destination.IsDoubleStackSlot()) << destination;
5704      MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex());
5705    }
5706  } else if (source.IsConstant()) {
5707    HConstant* constant = source.GetConstant();
5708    if (constant->IsIntConstant() || constant->IsNullConstant()) {
5709      int32_t value = CodeGenerator::GetInt32ValueOf(constant);
5710      if (destination.IsRegister()) {
5711        if (value == 0) {
5712          __ xorl(destination.AsRegister<Register>(), destination.AsRegister<Register>());
5713        } else {
5714          __ movl(destination.AsRegister<Register>(), Immediate(value));
5715        }
5716      } else {
5717        DCHECK(destination.IsStackSlot()) << destination;
5718        __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
5719      }
5720    } else if (constant->IsFloatConstant()) {
5721      float fp_value = constant->AsFloatConstant()->GetValue();
5722      int32_t value = bit_cast<int32_t, float>(fp_value);
5723      Immediate imm(value);
5724      if (destination.IsFpuRegister()) {
5725        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
5726        if (value == 0) {
5727          // Easy handling of 0.0.
5728          __ xorps(dest, dest);
5729        } else {
5730          ScratchRegisterScope ensure_scratch(
5731              this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5732          Register temp = static_cast<Register>(ensure_scratch.GetRegister());
5733          __ movl(temp, Immediate(value));
5734          __ movd(dest, temp);
5735        }
5736      } else {
5737        DCHECK(destination.IsStackSlot()) << destination;
5738        __ movl(Address(ESP, destination.GetStackIndex()), imm);
5739      }
5740    } else if (constant->IsLongConstant()) {
5741      int64_t value = constant->AsLongConstant()->GetValue();
5742      int32_t low_value = Low32Bits(value);
5743      int32_t high_value = High32Bits(value);
5744      Immediate low(low_value);
5745      Immediate high(high_value);
5746      if (destination.IsDoubleStackSlot()) {
5747        __ movl(Address(ESP, destination.GetStackIndex()), low);
5748        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
5749      } else {
5750        __ movl(destination.AsRegisterPairLow<Register>(), low);
5751        __ movl(destination.AsRegisterPairHigh<Register>(), high);
5752      }
5753    } else {
5754      DCHECK(constant->IsDoubleConstant());
5755      double dbl_value = constant->AsDoubleConstant()->GetValue();
5756      int64_t value = bit_cast<int64_t, double>(dbl_value);
5757      int32_t low_value = Low32Bits(value);
5758      int32_t high_value = High32Bits(value);
5759      Immediate low(low_value);
5760      Immediate high(high_value);
5761      if (destination.IsFpuRegister()) {
5762        XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
5763        if (value == 0) {
5764          // Easy handling of 0.0.
5765          __ xorpd(dest, dest);
5766        } else {
5767          __ pushl(high);
5768          __ pushl(low);
5769          __ movsd(dest, Address(ESP, 0));
5770          __ addl(ESP, Immediate(8));
5771        }
5772      } else {
5773        DCHECK(destination.IsDoubleStackSlot()) << destination;
5774        __ movl(Address(ESP, destination.GetStackIndex()), low);
5775        __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
5776      }
5777    }
5778  } else {
5779    LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
5780  }
5781}
5782
5783void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
5784  Register suggested_scratch = reg == EAX ? EBX : EAX;
5785  ScratchRegisterScope ensure_scratch(
5786      this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
5787
5788  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5789  __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
5790  __ movl(Address(ESP, mem + stack_offset), reg);
5791  __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
5792}
5793
5794void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) {
5795  ScratchRegisterScope ensure_scratch(
5796      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5797
5798  Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5799  int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5800  __ movl(temp_reg, Address(ESP, mem + stack_offset));
5801  __ movss(Address(ESP, mem + stack_offset), reg);
5802  __ movd(reg, temp_reg);
5803}
5804
5805void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
5806  ScratchRegisterScope ensure_scratch1(
5807      this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5808
5809  Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
5810  ScratchRegisterScope ensure_scratch2(
5811      this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
5812
5813  int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
5814  stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
5815  __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
5816  __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
5817  __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
5818  __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
5819}
5820
5821void ParallelMoveResolverX86::EmitSwap(size_t index) {
5822  MoveOperands* move = moves_[index];
5823  Location source = move->GetSource();
5824  Location destination = move->GetDestination();
5825
5826  if (source.IsRegister() && destination.IsRegister()) {
5827    // Use XOR swap algorithm to avoid serializing XCHG instruction or using a temporary.
5828    DCHECK_NE(destination.AsRegister<Register>(), source.AsRegister<Register>());
5829    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5830    __ xorl(source.AsRegister<Register>(), destination.AsRegister<Register>());
5831    __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5832  } else if (source.IsRegister() && destination.IsStackSlot()) {
5833    Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
5834  } else if (source.IsStackSlot() && destination.IsRegister()) {
5835    Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
5836  } else if (source.IsStackSlot() && destination.IsStackSlot()) {
5837    Exchange(destination.GetStackIndex(), source.GetStackIndex());
5838  } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
5839    // Use XOR Swap algorithm to avoid a temporary.
5840    DCHECK_NE(source.reg(), destination.reg());
5841    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5842    __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
5843    __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5844  } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
5845    Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
5846  } else if (destination.IsFpuRegister() && source.IsStackSlot()) {
5847    Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
5848  } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
5849    // Take advantage of the 16 bytes in the XMM register.
5850    XmmRegister reg = source.AsFpuRegister<XmmRegister>();
5851    Address stack(ESP, destination.GetStackIndex());
5852    // Load the double into the high doubleword.
5853    __ movhpd(reg, stack);
5854
5855    // Store the low double into the destination.
5856    __ movsd(stack, reg);
5857
5858    // Move the high double to the low double.
5859    __ psrldq(reg, Immediate(8));
5860  } else if (destination.IsFpuRegister() && source.IsDoubleStackSlot()) {
5861    // Take advantage of the 16 bytes in the XMM register.
5862    XmmRegister reg = destination.AsFpuRegister<XmmRegister>();
5863    Address stack(ESP, source.GetStackIndex());
5864    // Load the double into the high doubleword.
5865    __ movhpd(reg, stack);
5866
5867    // Store the low double into the destination.
5868    __ movsd(stack, reg);
5869
5870    // Move the high double to the low double.
5871    __ psrldq(reg, Immediate(8));
5872  } else if (destination.IsDoubleStackSlot() && source.IsDoubleStackSlot()) {
5873    Exchange(destination.GetStackIndex(), source.GetStackIndex());
5874    Exchange(destination.GetHighStackIndex(kX86WordSize), source.GetHighStackIndex(kX86WordSize));
5875  } else {
5876    LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination;
5877  }
5878}
5879
5880void ParallelMoveResolverX86::SpillScratch(int reg) {
5881  __ pushl(static_cast<Register>(reg));
5882}
5883
5884void ParallelMoveResolverX86::RestoreScratch(int reg) {
5885  __ popl(static_cast<Register>(reg));
5886}
5887
5888void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
5889  InvokeRuntimeCallingConvention calling_convention;
5890  CodeGenerator::CreateLoadClassLocationSummary(
5891      cls,
5892      Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
5893      Location::RegisterLocation(EAX),
5894      /* code_generator_supports_read_barrier */ true);
5895}
5896
5897void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
5898  LocationSummary* locations = cls->GetLocations();
5899  if (cls->NeedsAccessCheck()) {
5900    codegen_->MoveConstant(locations->GetTemp(0), cls->GetTypeIndex());
5901    codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInitializeTypeAndVerifyAccess),
5902                            cls,
5903                            cls->GetDexPc(),
5904                            nullptr);
5905    CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>();
5906    return;
5907  }
5908
5909  Location out_loc = locations->Out();
5910  Register out = out_loc.AsRegister<Register>();
5911  Register current_method = locations->InAt(0).AsRegister<Register>();
5912
5913  if (cls->IsReferrersClass()) {
5914    DCHECK(!cls->CanCallRuntime());
5915    DCHECK(!cls->MustGenerateClinitCheck());
5916    // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
5917    GenerateGcRootFieldLoad(
5918        cls, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
5919  } else {
5920    // /* GcRoot<mirror::Class>[] */ out =
5921    //        current_method.ptr_sized_fields_->dex_cache_resolved_types_
5922    __ movl(out, Address(current_method,
5923                         ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value()));
5924    // /* GcRoot<mirror::Class> */ out = out[type_index]
5925    GenerateGcRootFieldLoad(cls, out_loc, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex()));
5926
5927    if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) {
5928      DCHECK(cls->CanCallRuntime());
5929      SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
5930          cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
5931      codegen_->AddSlowPath(slow_path);
5932
5933      if (!cls->IsInDexCache()) {
5934        __ testl(out, out);
5935        __ j(kEqual, slow_path->GetEntryLabel());
5936      }
5937
5938      if (cls->MustGenerateClinitCheck()) {
5939        GenerateClassInitializationCheck(slow_path, out);
5940      } else {
5941        __ Bind(slow_path->GetExitLabel());
5942      }
5943    }
5944  }
5945}
5946
5947void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
5948  LocationSummary* locations =
5949      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
5950  locations->SetInAt(0, Location::RequiresRegister());
5951  if (check->HasUses()) {
5952    locations->SetOut(Location::SameAsFirstInput());
5953  }
5954}
5955
5956void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
5957  // We assume the class to not be null.
5958  SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
5959      check->GetLoadClass(), check, check->GetDexPc(), true);
5960  codegen_->AddSlowPath(slow_path);
5961  GenerateClassInitializationCheck(slow_path,
5962                                   check->GetLocations()->InAt(0).AsRegister<Register>());
5963}
5964
5965void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
5966    SlowPathCode* slow_path, Register class_reg) {
5967  __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
5968          Immediate(mirror::Class::kStatusInitialized));
5969  __ j(kLess, slow_path->GetEntryLabel());
5970  __ Bind(slow_path->GetExitLabel());
5971  // No need for memory fence, thanks to the X86 memory model.
5972}
5973
5974void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
5975  LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier)
5976      ? LocationSummary::kCallOnSlowPath
5977      : LocationSummary::kNoCall;
5978  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
5979  locations->SetInAt(0, Location::RequiresRegister());
5980  locations->SetOut(Location::RequiresRegister());
5981}
5982
5983void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
5984  LocationSummary* locations = load->GetLocations();
5985  Location out_loc = locations->Out();
5986  Register out = out_loc.AsRegister<Register>();
5987  Register current_method = locations->InAt(0).AsRegister<Register>();
5988
5989  // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
5990  GenerateGcRootFieldLoad(
5991      load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
5992  // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
5993  __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
5994  // /* GcRoot<mirror::String> */ out = out[string_index]
5995  GenerateGcRootFieldLoad(
5996      load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex()));
5997
5998  if (!load->IsInDexCache()) {
5999    SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
6000    codegen_->AddSlowPath(slow_path);
6001    __ testl(out, out);
6002    __ j(kEqual, slow_path->GetEntryLabel());
6003    __ Bind(slow_path->GetExitLabel());
6004  }
6005}
6006
6007static Address GetExceptionTlsAddress() {
6008  return Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
6009}
6010
6011void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
6012  LocationSummary* locations =
6013      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
6014  locations->SetOut(Location::RequiresRegister());
6015}
6016
6017void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
6018  __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), GetExceptionTlsAddress());
6019}
6020
6021void LocationsBuilderX86::VisitClearException(HClearException* clear) {
6022  new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
6023}
6024
6025void InstructionCodeGeneratorX86::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
6026  __ fs()->movl(GetExceptionTlsAddress(), Immediate(0));
6027}
6028
6029void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
6030  LocationSummary* locations =
6031      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
6032  InvokeRuntimeCallingConvention calling_convention;
6033  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
6034}
6035
6036void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
6037  codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pDeliverException),
6038                          instruction,
6039                          instruction->GetDexPc(),
6040                          nullptr);
6041  CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>();
6042}
6043
6044static bool TypeCheckNeedsATemporary(TypeCheckKind type_check_kind) {
6045  return kEmitCompilerReadBarrier &&
6046      (kUseBakerReadBarrier ||
6047       type_check_kind == TypeCheckKind::kAbstractClassCheck ||
6048       type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
6049       type_check_kind == TypeCheckKind::kArrayObjectCheck);
6050}
6051
6052void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
6053  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
6054  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
6055  switch (type_check_kind) {
6056    case TypeCheckKind::kExactCheck:
6057    case TypeCheckKind::kAbstractClassCheck:
6058    case TypeCheckKind::kClassHierarchyCheck:
6059    case TypeCheckKind::kArrayObjectCheck:
6060      call_kind =
6061          kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall;
6062      break;
6063    case TypeCheckKind::kArrayCheck:
6064    case TypeCheckKind::kUnresolvedCheck:
6065    case TypeCheckKind::kInterfaceCheck:
6066      call_kind = LocationSummary::kCallOnSlowPath;
6067      break;
6068  }
6069
6070  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
6071  locations->SetInAt(0, Location::RequiresRegister());
6072  locations->SetInAt(1, Location::Any());
6073  // Note that TypeCheckSlowPathX86 uses this "out" register too.
6074  locations->SetOut(Location::RequiresRegister());
6075  // When read barriers are enabled, we need a temporary register for
6076  // some cases.
6077  if (TypeCheckNeedsATemporary(type_check_kind)) {
6078    locations->AddTemp(Location::RequiresRegister());
6079  }
6080}
6081
6082void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
6083  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
6084  LocationSummary* locations = instruction->GetLocations();
6085  Location obj_loc = locations->InAt(0);
6086  Register obj = obj_loc.AsRegister<Register>();
6087  Location cls = locations->InAt(1);
6088  Location out_loc = locations->Out();
6089  Register out = out_loc.AsRegister<Register>();
6090  Location maybe_temp_loc = TypeCheckNeedsATemporary(type_check_kind) ?
6091      locations->GetTemp(0) :
6092      Location::NoLocation();
6093  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
6094  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
6095  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
6096  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
6097  SlowPathCode* slow_path = nullptr;
6098  NearLabel done, zero;
6099
6100  // Return 0 if `obj` is null.
6101  // Avoid null check if we know obj is not null.
6102  if (instruction->MustDoNullCheck()) {
6103    __ testl(obj, obj);
6104    __ j(kEqual, &zero);
6105  }
6106
6107  // /* HeapReference<Class> */ out = obj->klass_
6108  GenerateReferenceLoadTwoRegisters(instruction, out_loc, obj_loc, class_offset, maybe_temp_loc);
6109
6110  switch (type_check_kind) {
6111    case TypeCheckKind::kExactCheck: {
6112      if (cls.IsRegister()) {
6113        __ cmpl(out, cls.AsRegister<Register>());
6114      } else {
6115        DCHECK(cls.IsStackSlot()) << cls;
6116        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
6117      }
6118
6119      // Classes must be equal for the instanceof to succeed.
6120      __ j(kNotEqual, &zero);
6121      __ movl(out, Immediate(1));
6122      __ jmp(&done);
6123      break;
6124    }
6125
6126    case TypeCheckKind::kAbstractClassCheck: {
6127      // If the class is abstract, we eagerly fetch the super class of the
6128      // object to avoid doing a comparison we know will fail.
6129      NearLabel loop;
6130      __ Bind(&loop);
6131      // /* HeapReference<Class> */ out = out->super_class_
6132      GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, maybe_temp_loc);
6133      __ testl(out, out);
6134      // If `out` is null, we use it for the result, and jump to `done`.
6135      __ j(kEqual, &done);
6136      if (cls.IsRegister()) {
6137        __ cmpl(out, cls.AsRegister<Register>());
6138      } else {
6139        DCHECK(cls.IsStackSlot()) << cls;
6140        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
6141      }
6142      __ j(kNotEqual, &loop);
6143      __ movl(out, Immediate(1));
6144      if (zero.IsLinked()) {
6145        __ jmp(&done);
6146      }
6147      break;
6148    }
6149
6150    case TypeCheckKind::kClassHierarchyCheck: {
6151      // Walk over the class hierarchy to find a match.
6152      NearLabel loop, success;
6153      __ Bind(&loop);
6154      if (cls.IsRegister()) {
6155        __ cmpl(out, cls.AsRegister<Register>());
6156      } else {
6157        DCHECK(cls.IsStackSlot()) << cls;
6158        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
6159      }
6160      __ j(kEqual, &success);
6161      // /* HeapReference<Class> */ out = out->super_class_
6162      GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, maybe_temp_loc);
6163      __ testl(out, out);
6164      __ j(kNotEqual, &loop);
6165      // If `out` is null, we use it for the result, and jump to `done`.
6166      __ jmp(&done);
6167      __ Bind(&success);
6168      __ movl(out, Immediate(1));
6169      if (zero.IsLinked()) {
6170        __ jmp(&done);
6171      }
6172      break;
6173    }
6174
6175    case TypeCheckKind::kArrayObjectCheck: {
6176      // Do an exact check.
6177      NearLabel exact_check;
6178      if (cls.IsRegister()) {
6179        __ cmpl(out, cls.AsRegister<Register>());
6180      } else {
6181        DCHECK(cls.IsStackSlot()) << cls;
6182        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
6183      }
6184      __ j(kEqual, &exact_check);
6185      // Otherwise, we need to check that the object's class is a non-primitive array.
6186      // /* HeapReference<Class> */ out = out->component_type_
6187      GenerateReferenceLoadOneRegister(instruction, out_loc, component_offset, maybe_temp_loc);
6188      __ testl(out, out);
6189      // If `out` is null, we use it for the result, and jump to `done`.
6190      __ j(kEqual, &done);
6191      __ cmpw(Address(out, primitive_offset), Immediate(Primitive::kPrimNot));
6192      __ j(kNotEqual, &zero);
6193      __ Bind(&exact_check);
6194      __ movl(out, Immediate(1));
6195      __ jmp(&done);
6196      break;
6197    }
6198
6199    case TypeCheckKind::kArrayCheck: {
6200      if (cls.IsRegister()) {
6201        __ cmpl(out, cls.AsRegister<Register>());
6202      } else {
6203        DCHECK(cls.IsStackSlot()) << cls;
6204        __ cmpl(out, Address(ESP, cls.GetStackIndex()));
6205      }
6206      DCHECK(locations->OnlyCallsOnSlowPath());
6207      slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
6208                                                                    /* is_fatal */ false);
6209      codegen_->AddSlowPath(slow_path);
6210      __ j(kNotEqual, slow_path->GetEntryLabel());
6211      __ movl(out, Immediate(1));
6212      if (zero.IsLinked()) {
6213        __ jmp(&done);
6214      }
6215      break;
6216    }
6217
6218    case TypeCheckKind::kUnresolvedCheck:
6219    case TypeCheckKind::kInterfaceCheck: {
6220      // Note that we indeed only call on slow path, but we always go
6221      // into the slow path for the unresolved and interface check
6222      // cases.
6223      //
6224      // We cannot directly call the InstanceofNonTrivial runtime
6225      // entry point without resorting to a type checking slow path
6226      // here (i.e. by calling InvokeRuntime directly), as it would
6227      // require to assign fixed registers for the inputs of this
6228      // HInstanceOf instruction (following the runtime calling
6229      // convention), which might be cluttered by the potential first
6230      // read barrier emission at the beginning of this method.
6231      //
6232      // TODO: Introduce a new runtime entry point taking the object
6233      // to test (instead of its class) as argument, and let it deal
6234      // with the read barrier issues. This will let us refactor this
6235      // case of the `switch` code as it was previously (with a direct
6236      // call to the runtime not using a type checking slow path).
6237      // This should also be beneficial for the other cases above.
6238      DCHECK(locations->OnlyCallsOnSlowPath());
6239      slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
6240                                                                    /* is_fatal */ false);
6241      codegen_->AddSlowPath(slow_path);
6242      __ jmp(slow_path->GetEntryLabel());
6243      if (zero.IsLinked()) {
6244        __ jmp(&done);
6245      }
6246      break;
6247    }
6248  }
6249
6250  if (zero.IsLinked()) {
6251    __ Bind(&zero);
6252    __ xorl(out, out);
6253  }
6254
6255  if (done.IsLinked()) {
6256    __ Bind(&done);
6257  }
6258
6259  if (slow_path != nullptr) {
6260    __ Bind(slow_path->GetExitLabel());
6261  }
6262}
6263
6264void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
6265  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
6266  bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
6267  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
6268  switch (type_check_kind) {
6269    case TypeCheckKind::kExactCheck:
6270    case TypeCheckKind::kAbstractClassCheck:
6271    case TypeCheckKind::kClassHierarchyCheck:
6272    case TypeCheckKind::kArrayObjectCheck:
6273      call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ?
6274          LocationSummary::kCallOnSlowPath :
6275          LocationSummary::kNoCall;  // In fact, call on a fatal (non-returning) slow path.
6276      break;
6277    case TypeCheckKind::kArrayCheck:
6278    case TypeCheckKind::kUnresolvedCheck:
6279    case TypeCheckKind::kInterfaceCheck:
6280      call_kind = LocationSummary::kCallOnSlowPath;
6281      break;
6282  }
6283  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
6284  locations->SetInAt(0, Location::RequiresRegister());
6285  locations->SetInAt(1, Location::Any());
6286  // Note that TypeCheckSlowPathX86 uses this "temp" register too.
6287  locations->AddTemp(Location::RequiresRegister());
6288  // When read barriers are enabled, we need an additional temporary
6289  // register for some cases.
6290  if (TypeCheckNeedsATemporary(type_check_kind)) {
6291    locations->AddTemp(Location::RequiresRegister());
6292  }
6293}
6294
6295void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
6296  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
6297  LocationSummary* locations = instruction->GetLocations();
6298  Location obj_loc = locations->InAt(0);
6299  Register obj = obj_loc.AsRegister<Register>();
6300  Location cls = locations->InAt(1);
6301  Location temp_loc = locations->GetTemp(0);
6302  Register temp = temp_loc.AsRegister<Register>();
6303  Location maybe_temp2_loc = TypeCheckNeedsATemporary(type_check_kind) ?
6304      locations->GetTemp(1) :
6305      Location::NoLocation();
6306  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
6307  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
6308  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
6309  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
6310
6311  bool is_type_check_slow_path_fatal =
6312      (type_check_kind == TypeCheckKind::kExactCheck ||
6313       type_check_kind == TypeCheckKind::kAbstractClassCheck ||
6314       type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
6315       type_check_kind == TypeCheckKind::kArrayObjectCheck) &&
6316      !instruction->CanThrowIntoCatchBlock();
6317  SlowPathCode* type_check_slow_path =
6318      new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
6319                                                        is_type_check_slow_path_fatal);
6320  codegen_->AddSlowPath(type_check_slow_path);
6321
6322  NearLabel done;
6323  // Avoid null check if we know obj is not null.
6324  if (instruction->MustDoNullCheck()) {
6325    __ testl(obj, obj);
6326    __ j(kEqual, &done);
6327  }
6328
6329  // /* HeapReference<Class> */ temp = obj->klass_
6330  GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
6331
6332  switch (type_check_kind) {
6333    case TypeCheckKind::kExactCheck:
6334    case TypeCheckKind::kArrayCheck: {
6335      if (cls.IsRegister()) {
6336        __ cmpl(temp, cls.AsRegister<Register>());
6337      } else {
6338        DCHECK(cls.IsStackSlot()) << cls;
6339        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6340      }
6341      // Jump to slow path for throwing the exception or doing a
6342      // more involved array check.
6343      __ j(kNotEqual, type_check_slow_path->GetEntryLabel());
6344      break;
6345    }
6346
6347    case TypeCheckKind::kAbstractClassCheck: {
6348      // If the class is abstract, we eagerly fetch the super class of the
6349      // object to avoid doing a comparison we know will fail.
6350      NearLabel loop, compare_classes;
6351      __ Bind(&loop);
6352      // /* HeapReference<Class> */ temp = temp->super_class_
6353      GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, maybe_temp2_loc);
6354
6355      // If the class reference currently in `temp` is not null, jump
6356      // to the `compare_classes` label to compare it with the checked
6357      // class.
6358      __ testl(temp, temp);
6359      __ j(kNotEqual, &compare_classes);
6360      // Otherwise, jump to the slow path to throw the exception.
6361      //
6362      // But before, move back the object's class into `temp` before
6363      // going into the slow path, as it has been overwritten in the
6364      // meantime.
6365      // /* HeapReference<Class> */ temp = obj->klass_
6366      GenerateReferenceLoadTwoRegisters(
6367          instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
6368      __ jmp(type_check_slow_path->GetEntryLabel());
6369
6370      __ Bind(&compare_classes);
6371      if (cls.IsRegister()) {
6372        __ cmpl(temp, cls.AsRegister<Register>());
6373      } else {
6374        DCHECK(cls.IsStackSlot()) << cls;
6375        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6376      }
6377      __ j(kNotEqual, &loop);
6378      break;
6379    }
6380
6381    case TypeCheckKind::kClassHierarchyCheck: {
6382      // Walk over the class hierarchy to find a match.
6383      NearLabel loop;
6384      __ Bind(&loop);
6385      if (cls.IsRegister()) {
6386        __ cmpl(temp, cls.AsRegister<Register>());
6387      } else {
6388        DCHECK(cls.IsStackSlot()) << cls;
6389        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6390      }
6391      __ j(kEqual, &done);
6392
6393      // /* HeapReference<Class> */ temp = temp->super_class_
6394      GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, maybe_temp2_loc);
6395
6396      // If the class reference currently in `temp` is not null, jump
6397      // back at the beginning of the loop.
6398      __ testl(temp, temp);
6399      __ j(kNotEqual, &loop);
6400      // Otherwise, jump to the slow path to throw the exception.
6401      //
6402      // But before, move back the object's class into `temp` before
6403      // going into the slow path, as it has been overwritten in the
6404      // meantime.
6405      // /* HeapReference<Class> */ temp = obj->klass_
6406      GenerateReferenceLoadTwoRegisters(
6407          instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
6408      __ jmp(type_check_slow_path->GetEntryLabel());
6409      break;
6410    }
6411
6412    case TypeCheckKind::kArrayObjectCheck: {
6413      // Do an exact check.
6414      NearLabel check_non_primitive_component_type;
6415      if (cls.IsRegister()) {
6416        __ cmpl(temp, cls.AsRegister<Register>());
6417      } else {
6418        DCHECK(cls.IsStackSlot()) << cls;
6419        __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6420      }
6421      __ j(kEqual, &done);
6422
6423      // Otherwise, we need to check that the object's class is a non-primitive array.
6424      // /* HeapReference<Class> */ temp = temp->component_type_
6425      GenerateReferenceLoadOneRegister(instruction, temp_loc, component_offset, maybe_temp2_loc);
6426
6427      // If the component type is not null (i.e. the object is indeed
6428      // an array), jump to label `check_non_primitive_component_type`
6429      // to further check that this component type is not a primitive
6430      // type.
6431      __ testl(temp, temp);
6432      __ j(kNotEqual, &check_non_primitive_component_type);
6433      // Otherwise, jump to the slow path to throw the exception.
6434      //
6435      // But before, move back the object's class into `temp` before
6436      // going into the slow path, as it has been overwritten in the
6437      // meantime.
6438      // /* HeapReference<Class> */ temp = obj->klass_
6439      GenerateReferenceLoadTwoRegisters(
6440          instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
6441      __ jmp(type_check_slow_path->GetEntryLabel());
6442
6443      __ Bind(&check_non_primitive_component_type);
6444      __ cmpw(Address(temp, primitive_offset), Immediate(Primitive::kPrimNot));
6445      __ j(kEqual, &done);
6446      // Same comment as above regarding `temp` and the slow path.
6447      // /* HeapReference<Class> */ temp = obj->klass_
6448      GenerateReferenceLoadTwoRegisters(
6449          instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
6450      __ jmp(type_check_slow_path->GetEntryLabel());
6451      break;
6452    }
6453
6454    case TypeCheckKind::kUnresolvedCheck:
6455    case TypeCheckKind::kInterfaceCheck:
6456      // We always go into the type check slow path for the unresolved
6457      // and interface check cases.
6458      //
6459      // We cannot directly call the CheckCast runtime entry point
6460      // without resorting to a type checking slow path here (i.e. by
6461      // calling InvokeRuntime directly), as it would require to
6462      // assign fixed registers for the inputs of this HInstanceOf
6463      // instruction (following the runtime calling convention), which
6464      // might be cluttered by the potential first read barrier
6465      // emission at the beginning of this method.
6466      //
6467      // TODO: Introduce a new runtime entry point taking the object
6468      // to test (instead of its class) as argument, and let it deal
6469      // with the read barrier issues. This will let us refactor this
6470      // case of the `switch` code as it was previously (with a direct
6471      // call to the runtime not using a type checking slow path).
6472      // This should also be beneficial for the other cases above.
6473      __ jmp(type_check_slow_path->GetEntryLabel());
6474      break;
6475  }
6476  __ Bind(&done);
6477
6478  __ Bind(type_check_slow_path->GetExitLabel());
6479}
6480
6481void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
6482  LocationSummary* locations =
6483      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
6484  InvokeRuntimeCallingConvention calling_convention;
6485  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
6486}
6487
6488void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
6489  codegen_->InvokeRuntime(instruction->IsEnter() ? QUICK_ENTRY_POINT(pLockObject)
6490                                                 : QUICK_ENTRY_POINT(pUnlockObject),
6491                          instruction,
6492                          instruction->GetDexPc(),
6493                          nullptr);
6494  if (instruction->IsEnter()) {
6495    CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>();
6496  } else {
6497    CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>();
6498  }
6499}
6500
6501void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
6502void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
6503void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
6504
6505void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
6506  LocationSummary* locations =
6507      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
6508  DCHECK(instruction->GetResultType() == Primitive::kPrimInt
6509         || instruction->GetResultType() == Primitive::kPrimLong);
6510  locations->SetInAt(0, Location::RequiresRegister());
6511  locations->SetInAt(1, Location::Any());
6512  locations->SetOut(Location::SameAsFirstInput());
6513}
6514
6515void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
6516  HandleBitwiseOperation(instruction);
6517}
6518
6519void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
6520  HandleBitwiseOperation(instruction);
6521}
6522
6523void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
6524  HandleBitwiseOperation(instruction);
6525}
6526
6527void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
6528  LocationSummary* locations = instruction->GetLocations();
6529  Location first = locations->InAt(0);
6530  Location second = locations->InAt(1);
6531  DCHECK(first.Equals(locations->Out()));
6532
6533  if (instruction->GetResultType() == Primitive::kPrimInt) {
6534    if (second.IsRegister()) {
6535      if (instruction->IsAnd()) {
6536        __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
6537      } else if (instruction->IsOr()) {
6538        __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
6539      } else {
6540        DCHECK(instruction->IsXor());
6541        __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
6542      }
6543    } else if (second.IsConstant()) {
6544      if (instruction->IsAnd()) {
6545        __ andl(first.AsRegister<Register>(),
6546                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6547      } else if (instruction->IsOr()) {
6548        __ orl(first.AsRegister<Register>(),
6549               Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6550      } else {
6551        DCHECK(instruction->IsXor());
6552        __ xorl(first.AsRegister<Register>(),
6553                Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6554      }
6555    } else {
6556      if (instruction->IsAnd()) {
6557        __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6558      } else if (instruction->IsOr()) {
6559        __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6560      } else {
6561        DCHECK(instruction->IsXor());
6562        __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6563      }
6564    }
6565  } else {
6566    DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
6567    if (second.IsRegisterPair()) {
6568      if (instruction->IsAnd()) {
6569        __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6570        __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6571      } else if (instruction->IsOr()) {
6572        __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6573        __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6574      } else {
6575        DCHECK(instruction->IsXor());
6576        __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6577        __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6578      }
6579    } else if (second.IsDoubleStackSlot()) {
6580      if (instruction->IsAnd()) {
6581        __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6582        __ andl(first.AsRegisterPairHigh<Register>(),
6583                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6584      } else if (instruction->IsOr()) {
6585        __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6586        __ orl(first.AsRegisterPairHigh<Register>(),
6587                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6588      } else {
6589        DCHECK(instruction->IsXor());
6590        __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6591        __ xorl(first.AsRegisterPairHigh<Register>(),
6592                Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6593      }
6594    } else {
6595      DCHECK(second.IsConstant()) << second;
6596      int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
6597      int32_t low_value = Low32Bits(value);
6598      int32_t high_value = High32Bits(value);
6599      Immediate low(low_value);
6600      Immediate high(high_value);
6601      Register first_low = first.AsRegisterPairLow<Register>();
6602      Register first_high = first.AsRegisterPairHigh<Register>();
6603      if (instruction->IsAnd()) {
6604        if (low_value == 0) {
6605          __ xorl(first_low, first_low);
6606        } else if (low_value != -1) {
6607          __ andl(first_low, low);
6608        }
6609        if (high_value == 0) {
6610          __ xorl(first_high, first_high);
6611        } else if (high_value != -1) {
6612          __ andl(first_high, high);
6613        }
6614      } else if (instruction->IsOr()) {
6615        if (low_value != 0) {
6616          __ orl(first_low, low);
6617        }
6618        if (high_value != 0) {
6619          __ orl(first_high, high);
6620        }
6621      } else {
6622        DCHECK(instruction->IsXor());
6623        if (low_value != 0) {
6624          __ xorl(first_low, low);
6625        }
6626        if (high_value != 0) {
6627          __ xorl(first_high, high);
6628        }
6629      }
6630    }
6631  }
6632}
6633
6634void InstructionCodeGeneratorX86::GenerateReferenceLoadOneRegister(HInstruction* instruction,
6635                                                                   Location out,
6636                                                                   uint32_t offset,
6637                                                                   Location maybe_temp) {
6638  Register out_reg = out.AsRegister<Register>();
6639  if (kEmitCompilerReadBarrier) {
6640    DCHECK(maybe_temp.IsRegister()) << maybe_temp;
6641    if (kUseBakerReadBarrier) {
6642      // Load with fast path based Baker's read barrier.
6643      // /* HeapReference<Object> */ out = *(out + offset)
6644      codegen_->GenerateFieldLoadWithBakerReadBarrier(
6645          instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false);
6646    } else {
6647      // Load with slow path based read barrier.
6648      // Save the value of `out` into `maybe_temp` before overwriting it
6649      // in the following move operation, as we will need it for the
6650      // read barrier below.
6651      __ movl(maybe_temp.AsRegister<Register>(), out_reg);
6652      // /* HeapReference<Object> */ out = *(out + offset)
6653      __ movl(out_reg, Address(out_reg, offset));
6654      codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset);
6655    }
6656  } else {
6657    // Plain load with no read barrier.
6658    // /* HeapReference<Object> */ out = *(out + offset)
6659    __ movl(out_reg, Address(out_reg, offset));
6660    __ MaybeUnpoisonHeapReference(out_reg);
6661  }
6662}
6663
6664void InstructionCodeGeneratorX86::GenerateReferenceLoadTwoRegisters(HInstruction* instruction,
6665                                                                    Location out,
6666                                                                    Location obj,
6667                                                                    uint32_t offset,
6668                                                                    Location maybe_temp) {
6669  Register out_reg = out.AsRegister<Register>();
6670  Register obj_reg = obj.AsRegister<Register>();
6671  if (kEmitCompilerReadBarrier) {
6672    if (kUseBakerReadBarrier) {
6673      DCHECK(maybe_temp.IsRegister()) << maybe_temp;
6674      // Load with fast path based Baker's read barrier.
6675      // /* HeapReference<Object> */ out = *(obj + offset)
6676      codegen_->GenerateFieldLoadWithBakerReadBarrier(
6677          instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false);
6678    } else {
6679      // Load with slow path based read barrier.
6680      // /* HeapReference<Object> */ out = *(obj + offset)
6681      __ movl(out_reg, Address(obj_reg, offset));
6682      codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset);
6683    }
6684  } else {
6685    // Plain load with no read barrier.
6686    // /* HeapReference<Object> */ out = *(obj + offset)
6687    __ movl(out_reg, Address(obj_reg, offset));
6688    __ MaybeUnpoisonHeapReference(out_reg);
6689  }
6690}
6691
6692void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad(HInstruction* instruction,
6693                                                          Location root,
6694                                                          Register obj,
6695                                                          uint32_t offset) {
6696  Register root_reg = root.AsRegister<Register>();
6697  if (kEmitCompilerReadBarrier) {
6698    if (kUseBakerReadBarrier) {
6699      // Fast path implementation of art::ReadBarrier::BarrierForRoot when
6700      // Baker's read barrier are used:
6701      //
6702      //   root = obj.field;
6703      //   if (Thread::Current()->GetIsGcMarking()) {
6704      //     root = ReadBarrier::Mark(root)
6705      //   }
6706
6707      // /* GcRoot<mirror::Object> */ root = *(obj + offset)
6708      __ movl(root_reg, Address(obj, offset));
6709      static_assert(
6710          sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>),
6711          "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> "
6712          "have different sizes.");
6713      static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t),
6714                    "art::mirror::CompressedReference<mirror::Object> and int32_t "
6715                    "have different sizes.");
6716
6717      // Slow path used to mark the GC root `root`.
6718      SlowPathCode* slow_path =
6719          new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86(instruction, root, root);
6720      codegen_->AddSlowPath(slow_path);
6721
6722      __ fs()->cmpl(Address::Absolute(Thread::IsGcMarkingOffset<kX86WordSize>().Int32Value()),
6723                    Immediate(0));
6724      __ j(kNotEqual, slow_path->GetEntryLabel());
6725      __ Bind(slow_path->GetExitLabel());
6726    } else {
6727      // GC root loaded through a slow path for read barriers other
6728      // than Baker's.
6729      // /* GcRoot<mirror::Object>* */ root = obj + offset
6730      __ leal(root_reg, Address(obj, offset));
6731      // /* mirror::Object* */ root = root->Read()
6732      codegen_->GenerateReadBarrierForRootSlow(instruction, root, root);
6733    }
6734  } else {
6735    // Plain GC root load with no read barrier.
6736    // /* GcRoot<mirror::Object> */ root = *(obj + offset)
6737    __ movl(root_reg, Address(obj, offset));
6738    // Note that GC roots are not affected by heap poisoning, thus we
6739    // do not have to unpoison `root_reg` here.
6740  }
6741}
6742
6743void CodeGeneratorX86::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
6744                                                             Location ref,
6745                                                             Register obj,
6746                                                             uint32_t offset,
6747                                                             Location temp,
6748                                                             bool needs_null_check) {
6749  DCHECK(kEmitCompilerReadBarrier);
6750  DCHECK(kUseBakerReadBarrier);
6751
6752  // /* HeapReference<Object> */ ref = *(obj + offset)
6753  Address src(obj, offset);
6754  GenerateReferenceLoadWithBakerReadBarrier(instruction, ref, obj, src, temp, needs_null_check);
6755}
6756
6757void CodeGeneratorX86::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction,
6758                                                             Location ref,
6759                                                             Register obj,
6760                                                             uint32_t data_offset,
6761                                                             Location index,
6762                                                             Location temp,
6763                                                             bool needs_null_check) {
6764  DCHECK(kEmitCompilerReadBarrier);
6765  DCHECK(kUseBakerReadBarrier);
6766
6767  // /* HeapReference<Object> */ ref =
6768  //     *(obj + data_offset + index * sizeof(HeapReference<Object>))
6769  Address src = index.IsConstant() ?
6770      Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset) :
6771      Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset);
6772  GenerateReferenceLoadWithBakerReadBarrier(instruction, ref, obj, src, temp, needs_null_check);
6773}
6774
6775void CodeGeneratorX86::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction,
6776                                                                 Location ref,
6777                                                                 Register obj,
6778                                                                 const Address& src,
6779                                                                 Location temp,
6780                                                                 bool needs_null_check) {
6781  DCHECK(kEmitCompilerReadBarrier);
6782  DCHECK(kUseBakerReadBarrier);
6783
6784  // In slow path based read barriers, the read barrier call is
6785  // inserted after the original load. However, in fast path based
6786  // Baker's read barriers, we need to perform the load of
6787  // mirror::Object::monitor_ *before* the original reference load.
6788  // This load-load ordering is required by the read barrier.
6789  // The fast path/slow path (for Baker's algorithm) should look like:
6790  //
6791  //   uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
6792  //   lfence;  // Load fence or artificial data dependency to prevent load-load reordering
6793  //   HeapReference<Object> ref = *src;  // Original reference load.
6794  //   bool is_gray = (rb_state == ReadBarrier::gray_ptr_);
6795  //   if (is_gray) {
6796  //     ref = ReadBarrier::Mark(ref);  // Performed by runtime entrypoint slow path.
6797  //   }
6798  //
6799  // Note: the original implementation in ReadBarrier::Barrier is
6800  // slightly more complex as:
6801  // - it implements the load-load fence using a data dependency on
6802  //   the high-bits of rb_state, which are expected to be all zeroes
6803  //   (we use CodeGeneratorX86::GenerateMemoryBarrier instead here,
6804  //   which is a no-op thanks to the x86 memory model);
6805  // - it performs additional checks that we do not do here for
6806  //   performance reasons.
6807
6808  Register ref_reg = ref.AsRegister<Register>();
6809  Register temp_reg = temp.AsRegister<Register>();
6810  uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
6811
6812  // /* int32_t */ monitor = obj->monitor_
6813  __ movl(temp_reg, Address(obj, monitor_offset));
6814  if (needs_null_check) {
6815    MaybeRecordImplicitNullCheck(instruction);
6816  }
6817  // /* LockWord */ lock_word = LockWord(monitor)
6818  static_assert(sizeof(LockWord) == sizeof(int32_t),
6819                "art::LockWord and int32_t have different sizes.");
6820  // /* uint32_t */ rb_state = lock_word.ReadBarrierState()
6821  __ shrl(temp_reg, Immediate(LockWord::kReadBarrierStateShift));
6822  __ andl(temp_reg, Immediate(LockWord::kReadBarrierStateMask));
6823  static_assert(
6824      LockWord::kReadBarrierStateMask == ReadBarrier::rb_ptr_mask_,
6825      "art::LockWord::kReadBarrierStateMask is not equal to art::ReadBarrier::rb_ptr_mask_.");
6826
6827  // Load fence to prevent load-load reordering.
6828  // Note that this is a no-op, thanks to the x86 memory model.
6829  GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
6830
6831  // The actual reference load.
6832  // /* HeapReference<Object> */ ref = *src
6833  __ movl(ref_reg, src);
6834
6835  // Object* ref = ref_addr->AsMirrorPtr()
6836  __ MaybeUnpoisonHeapReference(ref_reg);
6837
6838  // Slow path used to mark the object `ref` when it is gray.
6839  SlowPathCode* slow_path =
6840      new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86(instruction, ref, ref);
6841  AddSlowPath(slow_path);
6842
6843  // if (rb_state == ReadBarrier::gray_ptr_)
6844  //   ref = ReadBarrier::Mark(ref);
6845  __ cmpl(temp_reg, Immediate(ReadBarrier::gray_ptr_));
6846  __ j(kEqual, slow_path->GetEntryLabel());
6847  __ Bind(slow_path->GetExitLabel());
6848}
6849
6850void CodeGeneratorX86::GenerateReadBarrierSlow(HInstruction* instruction,
6851                                               Location out,
6852                                               Location ref,
6853                                               Location obj,
6854                                               uint32_t offset,
6855                                               Location index) {
6856  DCHECK(kEmitCompilerReadBarrier);
6857
6858  // Insert a slow path based read barrier *after* the reference load.
6859  //
6860  // If heap poisoning is enabled, the unpoisoning of the loaded
6861  // reference will be carried out by the runtime within the slow
6862  // path.
6863  //
6864  // Note that `ref` currently does not get unpoisoned (when heap
6865  // poisoning is enabled), which is alright as the `ref` argument is
6866  // not used by the artReadBarrierSlow entry point.
6867  //
6868  // TODO: Unpoison `ref` when it is used by artReadBarrierSlow.
6869  SlowPathCode* slow_path = new (GetGraph()->GetArena())
6870      ReadBarrierForHeapReferenceSlowPathX86(instruction, out, ref, obj, offset, index);
6871  AddSlowPath(slow_path);
6872
6873  __ jmp(slow_path->GetEntryLabel());
6874  __ Bind(slow_path->GetExitLabel());
6875}
6876
6877void CodeGeneratorX86::MaybeGenerateReadBarrierSlow(HInstruction* instruction,
6878                                                    Location out,
6879                                                    Location ref,
6880                                                    Location obj,
6881                                                    uint32_t offset,
6882                                                    Location index) {
6883  if (kEmitCompilerReadBarrier) {
6884    // Baker's read barriers shall be handled by the fast path
6885    // (CodeGeneratorX86::GenerateReferenceLoadWithBakerReadBarrier).
6886    DCHECK(!kUseBakerReadBarrier);
6887    // If heap poisoning is enabled, unpoisoning will be taken care of
6888    // by the runtime within the slow path.
6889    GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index);
6890  } else if (kPoisonHeapReferences) {
6891    __ UnpoisonHeapReference(out.AsRegister<Register>());
6892  }
6893}
6894
6895void CodeGeneratorX86::GenerateReadBarrierForRootSlow(HInstruction* instruction,
6896                                                      Location out,
6897                                                      Location root) {
6898  DCHECK(kEmitCompilerReadBarrier);
6899
6900  // Insert a slow path based read barrier *after* the GC root load.
6901  //
6902  // Note that GC roots are not affected by heap poisoning, so we do
6903  // not need to do anything special for this here.
6904  SlowPathCode* slow_path =
6905      new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathX86(instruction, out, root);
6906  AddSlowPath(slow_path);
6907
6908  __ jmp(slow_path->GetEntryLabel());
6909  __ Bind(slow_path->GetExitLabel());
6910}
6911
6912void LocationsBuilderX86::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
6913  // Nothing to do, this should be removed during prepare for register allocator.
6914  LOG(FATAL) << "Unreachable";
6915}
6916
6917void InstructionCodeGeneratorX86::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
6918  // Nothing to do, this should be removed during prepare for register allocator.
6919  LOG(FATAL) << "Unreachable";
6920}
6921
6922// Simple implementation of packed switch - generate cascaded compare/jumps.
6923void LocationsBuilderX86::VisitPackedSwitch(HPackedSwitch* switch_instr) {
6924  LocationSummary* locations =
6925      new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
6926  locations->SetInAt(0, Location::RequiresRegister());
6927}
6928
6929void InstructionCodeGeneratorX86::GenPackedSwitchWithCompares(Register value_reg,
6930                                                              int32_t lower_bound,
6931                                                              uint32_t num_entries,
6932                                                              HBasicBlock* switch_block,
6933                                                              HBasicBlock* default_block) {
6934  // Figure out the correct compare values and jump conditions.
6935  // Handle the first compare/branch as a special case because it might
6936  // jump to the default case.
6937  DCHECK_GT(num_entries, 2u);
6938  Condition first_condition;
6939  uint32_t index;
6940  const ArenaVector<HBasicBlock*>& successors = switch_block->GetSuccessors();
6941  if (lower_bound != 0) {
6942    first_condition = kLess;
6943    __ cmpl(value_reg, Immediate(lower_bound));
6944    __ j(first_condition, codegen_->GetLabelOf(default_block));
6945    __ j(kEqual, codegen_->GetLabelOf(successors[0]));
6946
6947    index = 1;
6948  } else {
6949    // Handle all the compare/jumps below.
6950    first_condition = kBelow;
6951    index = 0;
6952  }
6953
6954  // Handle the rest of the compare/jumps.
6955  for (; index + 1 < num_entries; index += 2) {
6956    int32_t compare_to_value = lower_bound + index + 1;
6957    __ cmpl(value_reg, Immediate(compare_to_value));
6958    // Jump to successors[index] if value < case_value[index].
6959    __ j(first_condition, codegen_->GetLabelOf(successors[index]));
6960    // Jump to successors[index + 1] if value == case_value[index + 1].
6961    __ j(kEqual, codegen_->GetLabelOf(successors[index + 1]));
6962  }
6963
6964  if (index != num_entries) {
6965    // There are an odd number of entries. Handle the last one.
6966    DCHECK_EQ(index + 1, num_entries);
6967    __ cmpl(value_reg, Immediate(lower_bound + index));
6968    __ j(kEqual, codegen_->GetLabelOf(successors[index]));
6969  }
6970
6971  // And the default for any other value.
6972  if (!codegen_->GoesToNextBlock(switch_block, default_block)) {
6973    __ jmp(codegen_->GetLabelOf(default_block));
6974  }
6975}
6976
6977void InstructionCodeGeneratorX86::VisitPackedSwitch(HPackedSwitch* switch_instr) {
6978  int32_t lower_bound = switch_instr->GetStartValue();
6979  uint32_t num_entries = switch_instr->GetNumEntries();
6980  LocationSummary* locations = switch_instr->GetLocations();
6981  Register value_reg = locations->InAt(0).AsRegister<Register>();
6982
6983  GenPackedSwitchWithCompares(value_reg,
6984                              lower_bound,
6985                              num_entries,
6986                              switch_instr->GetBlock(),
6987                              switch_instr->GetDefaultBlock());
6988}
6989
6990void LocationsBuilderX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) {
6991  LocationSummary* locations =
6992      new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
6993  locations->SetInAt(0, Location::RequiresRegister());
6994
6995  // Constant area pointer.
6996  locations->SetInAt(1, Location::RequiresRegister());
6997
6998  // And the temporary we need.
6999  locations->AddTemp(Location::RequiresRegister());
7000}
7001
7002void InstructionCodeGeneratorX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) {
7003  int32_t lower_bound = switch_instr->GetStartValue();
7004  uint32_t num_entries = switch_instr->GetNumEntries();
7005  LocationSummary* locations = switch_instr->GetLocations();
7006  Register value_reg = locations->InAt(0).AsRegister<Register>();
7007  HBasicBlock* default_block = switch_instr->GetDefaultBlock();
7008
7009  if (num_entries <= kPackedSwitchJumpTableThreshold) {
7010    GenPackedSwitchWithCompares(value_reg,
7011                                lower_bound,
7012                                num_entries,
7013                                switch_instr->GetBlock(),
7014                                default_block);
7015    return;
7016  }
7017
7018  // Optimizing has a jump area.
7019  Register temp_reg = locations->GetTemp(0).AsRegister<Register>();
7020  Register constant_area = locations->InAt(1).AsRegister<Register>();
7021
7022  // Remove the bias, if needed.
7023  if (lower_bound != 0) {
7024    __ leal(temp_reg, Address(value_reg, -lower_bound));
7025    value_reg = temp_reg;
7026  }
7027
7028  // Is the value in range?
7029  DCHECK_GE(num_entries, 1u);
7030  __ cmpl(value_reg, Immediate(num_entries - 1));
7031  __ j(kAbove, codegen_->GetLabelOf(default_block));
7032
7033  // We are in the range of the table.
7034  // Load (target-constant_area) from the jump table, indexing by the value.
7035  __ movl(temp_reg, codegen_->LiteralCaseTable(switch_instr, constant_area, value_reg));
7036
7037  // Compute the actual target address by adding in constant_area.
7038  __ addl(temp_reg, constant_area);
7039
7040  // And jump.
7041  __ jmp(temp_reg);
7042}
7043
7044void LocationsBuilderX86::VisitX86ComputeBaseMethodAddress(
7045    HX86ComputeBaseMethodAddress* insn) {
7046  LocationSummary* locations =
7047      new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
7048  locations->SetOut(Location::RequiresRegister());
7049}
7050
7051void InstructionCodeGeneratorX86::VisitX86ComputeBaseMethodAddress(
7052    HX86ComputeBaseMethodAddress* insn) {
7053  LocationSummary* locations = insn->GetLocations();
7054  Register reg = locations->Out().AsRegister<Register>();
7055
7056  // Generate call to next instruction.
7057  Label next_instruction;
7058  __ call(&next_instruction);
7059  __ Bind(&next_instruction);
7060
7061  // Remember this offset for later use with constant area.
7062  codegen_->SetMethodAddressOffset(GetAssembler()->CodeSize());
7063
7064  // Grab the return address off the stack.
7065  __ popl(reg);
7066}
7067
7068void LocationsBuilderX86::VisitX86LoadFromConstantTable(
7069    HX86LoadFromConstantTable* insn) {
7070  LocationSummary* locations =
7071      new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
7072
7073  locations->SetInAt(0, Location::RequiresRegister());
7074  locations->SetInAt(1, Location::ConstantLocation(insn->GetConstant()));
7075
7076  // If we don't need to be materialized, we only need the inputs to be set.
7077  if (insn->IsEmittedAtUseSite()) {
7078    return;
7079  }
7080
7081  switch (insn->GetType()) {
7082    case Primitive::kPrimFloat:
7083    case Primitive::kPrimDouble:
7084      locations->SetOut(Location::RequiresFpuRegister());
7085      break;
7086
7087    case Primitive::kPrimInt:
7088      locations->SetOut(Location::RequiresRegister());
7089      break;
7090
7091    default:
7092      LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
7093  }
7094}
7095
7096void InstructionCodeGeneratorX86::VisitX86LoadFromConstantTable(HX86LoadFromConstantTable* insn) {
7097  if (insn->IsEmittedAtUseSite()) {
7098    return;
7099  }
7100
7101  LocationSummary* locations = insn->GetLocations();
7102  Location out = locations->Out();
7103  Register const_area = locations->InAt(0).AsRegister<Register>();
7104  HConstant *value = insn->GetConstant();
7105
7106  switch (insn->GetType()) {
7107    case Primitive::kPrimFloat:
7108      __ movss(out.AsFpuRegister<XmmRegister>(),
7109               codegen_->LiteralFloatAddress(value->AsFloatConstant()->GetValue(), const_area));
7110      break;
7111
7112    case Primitive::kPrimDouble:
7113      __ movsd(out.AsFpuRegister<XmmRegister>(),
7114               codegen_->LiteralDoubleAddress(value->AsDoubleConstant()->GetValue(), const_area));
7115      break;
7116
7117    case Primitive::kPrimInt:
7118      __ movl(out.AsRegister<Register>(),
7119              codegen_->LiteralInt32Address(value->AsIntConstant()->GetValue(), const_area));
7120      break;
7121
7122    default:
7123      LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
7124  }
7125}
7126
7127/**
7128 * Class to handle late fixup of offsets into constant area.
7129 */
7130class RIPFixup : public AssemblerFixup, public ArenaObject<kArenaAllocCodeGenerator> {
7131 public:
7132  RIPFixup(CodeGeneratorX86& codegen, size_t offset)
7133      : codegen_(&codegen), offset_into_constant_area_(offset) {}
7134
7135 protected:
7136  void SetOffset(size_t offset) { offset_into_constant_area_ = offset; }
7137
7138  CodeGeneratorX86* codegen_;
7139
7140 private:
7141  void Process(const MemoryRegion& region, int pos) OVERRIDE {
7142    // Patch the correct offset for the instruction.  The place to patch is the
7143    // last 4 bytes of the instruction.
7144    // The value to patch is the distance from the offset in the constant area
7145    // from the address computed by the HX86ComputeBaseMethodAddress instruction.
7146    int32_t constant_offset = codegen_->ConstantAreaStart() + offset_into_constant_area_;
7147    int32_t relative_position = constant_offset - codegen_->GetMethodAddressOffset();;
7148
7149    // Patch in the right value.
7150    region.StoreUnaligned<int32_t>(pos - 4, relative_position);
7151  }
7152
7153  // Location in constant area that the fixup refers to.
7154  int32_t offset_into_constant_area_;
7155};
7156
7157/**
7158 * Class to handle late fixup of offsets to a jump table that will be created in the
7159 * constant area.
7160 */
7161class JumpTableRIPFixup : public RIPFixup {
7162 public:
7163  JumpTableRIPFixup(CodeGeneratorX86& codegen, HX86PackedSwitch* switch_instr)
7164      : RIPFixup(codegen, static_cast<size_t>(-1)), switch_instr_(switch_instr) {}
7165
7166  void CreateJumpTable() {
7167    X86Assembler* assembler = codegen_->GetAssembler();
7168
7169    // Ensure that the reference to the jump table has the correct offset.
7170    const int32_t offset_in_constant_table = assembler->ConstantAreaSize();
7171    SetOffset(offset_in_constant_table);
7172
7173    // The label values in the jump table are computed relative to the
7174    // instruction addressing the constant area.
7175    const int32_t relative_offset = codegen_->GetMethodAddressOffset();
7176
7177    // Populate the jump table with the correct values for the jump table.
7178    int32_t num_entries = switch_instr_->GetNumEntries();
7179    HBasicBlock* block = switch_instr_->GetBlock();
7180    const ArenaVector<HBasicBlock*>& successors = block->GetSuccessors();
7181    // The value that we want is the target offset - the position of the table.
7182    for (int32_t i = 0; i < num_entries; i++) {
7183      HBasicBlock* b = successors[i];
7184      Label* l = codegen_->GetLabelOf(b);
7185      DCHECK(l->IsBound());
7186      int32_t offset_to_block = l->Position() - relative_offset;
7187      assembler->AppendInt32(offset_to_block);
7188    }
7189  }
7190
7191 private:
7192  const HX86PackedSwitch* switch_instr_;
7193};
7194
7195void CodeGeneratorX86::Finalize(CodeAllocator* allocator) {
7196  // Generate the constant area if needed.
7197  X86Assembler* assembler = GetAssembler();
7198  if (!assembler->IsConstantAreaEmpty() || !fixups_to_jump_tables_.empty()) {
7199    // Align to 4 byte boundary to reduce cache misses, as the data is 4 and 8
7200    // byte values.
7201    assembler->Align(4, 0);
7202    constant_area_start_ = assembler->CodeSize();
7203
7204    // Populate any jump tables.
7205    for (auto jump_table : fixups_to_jump_tables_) {
7206      jump_table->CreateJumpTable();
7207    }
7208
7209    // And now add the constant area to the generated code.
7210    assembler->AddConstantArea();
7211  }
7212
7213  // And finish up.
7214  CodeGenerator::Finalize(allocator);
7215}
7216
7217Address CodeGeneratorX86::LiteralDoubleAddress(double v, Register reg) {
7218  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddDouble(v));
7219  return Address(reg, kDummy32BitOffset, fixup);
7220}
7221
7222Address CodeGeneratorX86::LiteralFloatAddress(float v, Register reg) {
7223  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddFloat(v));
7224  return Address(reg, kDummy32BitOffset, fixup);
7225}
7226
7227Address CodeGeneratorX86::LiteralInt32Address(int32_t v, Register reg) {
7228  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt32(v));
7229  return Address(reg, kDummy32BitOffset, fixup);
7230}
7231
7232Address CodeGeneratorX86::LiteralInt64Address(int64_t v, Register reg) {
7233  AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt64(v));
7234  return Address(reg, kDummy32BitOffset, fixup);
7235}
7236
7237void CodeGeneratorX86::Load32BitValue(Register dest, int32_t value) {
7238  if (value == 0) {
7239    __ xorl(dest, dest);
7240  } else {
7241    __ movl(dest, Immediate(value));
7242  }
7243}
7244
7245void CodeGeneratorX86::Compare32BitValue(Register dest, int32_t value) {
7246  if (value == 0) {
7247    __ testl(dest, dest);
7248  } else {
7249    __ cmpl(dest, Immediate(value));
7250  }
7251}
7252
7253Address CodeGeneratorX86::LiteralCaseTable(HX86PackedSwitch* switch_instr,
7254                                           Register reg,
7255                                           Register value) {
7256  // Create a fixup to be used to create and address the jump table.
7257  JumpTableRIPFixup* table_fixup =
7258      new (GetGraph()->GetArena()) JumpTableRIPFixup(*this, switch_instr);
7259
7260  // We have to populate the jump tables.
7261  fixups_to_jump_tables_.push_back(table_fixup);
7262
7263  // We want a scaled address, as we are extracting the correct offset from the table.
7264  return Address(reg, value, TIMES_4, kDummy32BitOffset, table_fixup);
7265}
7266
7267// TODO: target as memory.
7268void CodeGeneratorX86::MoveFromReturnRegister(Location target, Primitive::Type type) {
7269  if (!target.IsValid()) {
7270    DCHECK_EQ(type, Primitive::kPrimVoid);
7271    return;
7272  }
7273
7274  DCHECK_NE(type, Primitive::kPrimVoid);
7275
7276  Location return_loc = InvokeDexCallingConventionVisitorX86().GetReturnLocation(type);
7277  if (target.Equals(return_loc)) {
7278    return;
7279  }
7280
7281  // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged
7282  //       with the else branch.
7283  if (type == Primitive::kPrimLong) {
7284    HParallelMove parallel_move(GetGraph()->GetArena());
7285    parallel_move.AddMove(return_loc.ToLow(), target.ToLow(), Primitive::kPrimInt, nullptr);
7286    parallel_move.AddMove(return_loc.ToHigh(), target.ToHigh(), Primitive::kPrimInt, nullptr);
7287    GetMoveResolver()->EmitNativeCode(&parallel_move);
7288  } else {
7289    // Let the parallel move resolver take care of all of this.
7290    HParallelMove parallel_move(GetGraph()->GetArena());
7291    parallel_move.AddMove(return_loc, target, type, nullptr);
7292    GetMoveResolver()->EmitNativeCode(&parallel_move);
7293  }
7294}
7295
7296#undef __
7297
7298}  // namespace x86
7299}  // namespace art
7300