1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator.h"
18
19#ifdef ART_ENABLE_CODEGEN_arm
20#include "code_generator_arm.h"
21#endif
22
23#ifdef ART_ENABLE_CODEGEN_arm64
24#include "code_generator_arm64.h"
25#endif
26
27#ifdef ART_ENABLE_CODEGEN_x86
28#include "code_generator_x86.h"
29#endif
30
31#ifdef ART_ENABLE_CODEGEN_x86_64
32#include "code_generator_x86_64.h"
33#endif
34
35#ifdef ART_ENABLE_CODEGEN_mips
36#include "code_generator_mips.h"
37#endif
38
39#ifdef ART_ENABLE_CODEGEN_mips64
40#include "code_generator_mips64.h"
41#endif
42
43#include "bytecode_utils.h"
44#include "compiled_method.h"
45#include "dex/verified_method.h"
46#include "driver/compiler_driver.h"
47#include "graph_visualizer.h"
48#include "intrinsics.h"
49#include "leb128.h"
50#include "mirror/array-inl.h"
51#include "mirror/object_array-inl.h"
52#include "mirror/object_reference.h"
53#include "parallel_move_resolver.h"
54#include "ssa_liveness_analysis.h"
55#include "utils/assembler.h"
56
57namespace art {
58
59// Return whether a location is consistent with a type.
60static bool CheckType(Primitive::Type type, Location location) {
61  if (location.IsFpuRegister()
62      || (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresFpuRegister))) {
63    return (type == Primitive::kPrimFloat) || (type == Primitive::kPrimDouble);
64  } else if (location.IsRegister() ||
65             (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresRegister))) {
66    return Primitive::IsIntegralType(type) || (type == Primitive::kPrimNot);
67  } else if (location.IsRegisterPair()) {
68    return type == Primitive::kPrimLong;
69  } else if (location.IsFpuRegisterPair()) {
70    return type == Primitive::kPrimDouble;
71  } else if (location.IsStackSlot()) {
72    return (Primitive::IsIntegralType(type) && type != Primitive::kPrimLong)
73           || (type == Primitive::kPrimFloat)
74           || (type == Primitive::kPrimNot);
75  } else if (location.IsDoubleStackSlot()) {
76    return (type == Primitive::kPrimLong) || (type == Primitive::kPrimDouble);
77  } else if (location.IsConstant()) {
78    if (location.GetConstant()->IsIntConstant()) {
79      return Primitive::IsIntegralType(type) && (type != Primitive::kPrimLong);
80    } else if (location.GetConstant()->IsNullConstant()) {
81      return type == Primitive::kPrimNot;
82    } else if (location.GetConstant()->IsLongConstant()) {
83      return type == Primitive::kPrimLong;
84    } else if (location.GetConstant()->IsFloatConstant()) {
85      return type == Primitive::kPrimFloat;
86    } else {
87      return location.GetConstant()->IsDoubleConstant()
88          && (type == Primitive::kPrimDouble);
89    }
90  } else {
91    return location.IsInvalid() || (location.GetPolicy() == Location::kAny);
92  }
93}
94
95// Check that a location summary is consistent with an instruction.
96static bool CheckTypeConsistency(HInstruction* instruction) {
97  LocationSummary* locations = instruction->GetLocations();
98  if (locations == nullptr) {
99    return true;
100  }
101
102  if (locations->Out().IsUnallocated()
103      && (locations->Out().GetPolicy() == Location::kSameAsFirstInput)) {
104    DCHECK(CheckType(instruction->GetType(), locations->InAt(0)))
105        << instruction->GetType()
106        << " " << locations->InAt(0);
107  } else {
108    DCHECK(CheckType(instruction->GetType(), locations->Out()))
109        << instruction->GetType()
110        << " " << locations->Out();
111  }
112
113  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
114    DCHECK(CheckType(instruction->InputAt(i)->GetType(), locations->InAt(i)))
115      << instruction->InputAt(i)->GetType()
116      << " " << locations->InAt(i);
117  }
118
119  HEnvironment* environment = instruction->GetEnvironment();
120  for (size_t i = 0; i < instruction->EnvironmentSize(); ++i) {
121    if (environment->GetInstructionAt(i) != nullptr) {
122      Primitive::Type type = environment->GetInstructionAt(i)->GetType();
123      DCHECK(CheckType(type, environment->GetLocationAt(i)))
124        << type << " " << environment->GetLocationAt(i);
125    } else {
126      DCHECK(environment->GetLocationAt(i).IsInvalid())
127        << environment->GetLocationAt(i);
128    }
129  }
130  return true;
131}
132
133size_t CodeGenerator::GetCacheOffset(uint32_t index) {
134  return sizeof(GcRoot<mirror::Object>) * index;
135}
136
137size_t CodeGenerator::GetCachePointerOffset(uint32_t index) {
138  auto pointer_size = InstructionSetPointerSize(GetInstructionSet());
139  return pointer_size * index;
140}
141
142bool CodeGenerator::GoesToNextBlock(HBasicBlock* current, HBasicBlock* next) const {
143  DCHECK_EQ((*block_order_)[current_block_index_], current);
144  return GetNextBlockToEmit() == FirstNonEmptyBlock(next);
145}
146
147HBasicBlock* CodeGenerator::GetNextBlockToEmit() const {
148  for (size_t i = current_block_index_ + 1; i < block_order_->size(); ++i) {
149    HBasicBlock* block = (*block_order_)[i];
150    if (!block->IsSingleJump()) {
151      return block;
152    }
153  }
154  return nullptr;
155}
156
157HBasicBlock* CodeGenerator::FirstNonEmptyBlock(HBasicBlock* block) const {
158  while (block->IsSingleJump()) {
159    block = block->GetSuccessors()[0];
160  }
161  return block;
162}
163
164class DisassemblyScope {
165 public:
166  DisassemblyScope(HInstruction* instruction, const CodeGenerator& codegen)
167      : codegen_(codegen), instruction_(instruction), start_offset_(static_cast<size_t>(-1)) {
168    if (codegen_.GetDisassemblyInformation() != nullptr) {
169      start_offset_ = codegen_.GetAssembler().CodeSize();
170    }
171  }
172
173  ~DisassemblyScope() {
174    // We avoid building this data when we know it will not be used.
175    if (codegen_.GetDisassemblyInformation() != nullptr) {
176      codegen_.GetDisassemblyInformation()->AddInstructionInterval(
177          instruction_, start_offset_, codegen_.GetAssembler().CodeSize());
178    }
179  }
180
181 private:
182  const CodeGenerator& codegen_;
183  HInstruction* instruction_;
184  size_t start_offset_;
185};
186
187
188void CodeGenerator::GenerateSlowPaths() {
189  size_t code_start = 0;
190  for (const std::unique_ptr<SlowPathCode>& slow_path_unique_ptr : slow_paths_) {
191    SlowPathCode* slow_path = slow_path_unique_ptr.get();
192    current_slow_path_ = slow_path;
193    if (disasm_info_ != nullptr) {
194      code_start = GetAssembler()->CodeSize();
195    }
196    // Record the dex pc at start of slow path (required for java line number mapping).
197    MaybeRecordNativeDebugInfo(slow_path->GetInstruction(), slow_path->GetDexPc(), slow_path);
198    slow_path->EmitNativeCode(this);
199    if (disasm_info_ != nullptr) {
200      disasm_info_->AddSlowPathInterval(slow_path, code_start, GetAssembler()->CodeSize());
201    }
202  }
203  current_slow_path_ = nullptr;
204}
205
206void CodeGenerator::Compile(CodeAllocator* allocator) {
207  // The register allocator already called `InitializeCodeGeneration`,
208  // where the frame size has been computed.
209  DCHECK(block_order_ != nullptr);
210  Initialize();
211
212  HGraphVisitor* instruction_visitor = GetInstructionVisitor();
213  DCHECK_EQ(current_block_index_, 0u);
214
215  size_t frame_start = GetAssembler()->CodeSize();
216  GenerateFrameEntry();
217  DCHECK_EQ(GetAssembler()->cfi().GetCurrentCFAOffset(), static_cast<int>(frame_size_));
218  if (disasm_info_ != nullptr) {
219    disasm_info_->SetFrameEntryInterval(frame_start, GetAssembler()->CodeSize());
220  }
221
222  for (size_t e = block_order_->size(); current_block_index_ < e; ++current_block_index_) {
223    HBasicBlock* block = (*block_order_)[current_block_index_];
224    // Don't generate code for an empty block. Its predecessors will branch to its successor
225    // directly. Also, the label of that block will not be emitted, so this helps catch
226    // errors where we reference that label.
227    if (block->IsSingleJump()) continue;
228    Bind(block);
229    // This ensures that we have correct native line mapping for all native instructions.
230    // It is necessary to make stepping over a statement work. Otherwise, any initial
231    // instructions (e.g. moves) would be assumed to be the start of next statement.
232    MaybeRecordNativeDebugInfo(nullptr /* instruction */, block->GetDexPc());
233    for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) {
234      HInstruction* current = it.Current();
235      if (current->HasEnvironment()) {
236        // Create stackmap for HNativeDebugInfo or any instruction which calls native code.
237        // Note that we need correct mapping for the native PC of the call instruction,
238        // so the runtime's stackmap is not sufficient since it is at PC after the call.
239        MaybeRecordNativeDebugInfo(current, block->GetDexPc());
240      }
241      DisassemblyScope disassembly_scope(current, *this);
242      DCHECK(CheckTypeConsistency(current));
243      current->Accept(instruction_visitor);
244    }
245  }
246
247  GenerateSlowPaths();
248
249  // Emit catch stack maps at the end of the stack map stream as expected by the
250  // runtime exception handler.
251  if (graph_->HasTryCatch()) {
252    RecordCatchBlockInfo();
253  }
254
255  // Finalize instructions in assember;
256  Finalize(allocator);
257}
258
259void CodeGenerator::Finalize(CodeAllocator* allocator) {
260  size_t code_size = GetAssembler()->CodeSize();
261  uint8_t* buffer = allocator->Allocate(code_size);
262
263  MemoryRegion code(buffer, code_size);
264  GetAssembler()->FinalizeInstructions(code);
265}
266
267void CodeGenerator::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches ATTRIBUTE_UNUSED) {
268  // No linker patches by default.
269}
270
271void CodeGenerator::InitializeCodeGeneration(size_t number_of_spill_slots,
272                                             size_t maximum_number_of_live_core_registers,
273                                             size_t maximum_number_of_live_fpu_registers,
274                                             size_t number_of_out_slots,
275                                             const ArenaVector<HBasicBlock*>& block_order) {
276  block_order_ = &block_order;
277  DCHECK(!block_order.empty());
278  DCHECK(block_order[0] == GetGraph()->GetEntryBlock());
279  ComputeSpillMask();
280  first_register_slot_in_slow_path_ = (number_of_out_slots + number_of_spill_slots) * kVRegSize;
281
282  if (number_of_spill_slots == 0
283      && !HasAllocatedCalleeSaveRegisters()
284      && IsLeafMethod()
285      && !RequiresCurrentMethod()) {
286    DCHECK_EQ(maximum_number_of_live_core_registers, 0u);
287    DCHECK_EQ(maximum_number_of_live_fpu_registers, 0u);
288    SetFrameSize(CallPushesPC() ? GetWordSize() : 0);
289  } else {
290    SetFrameSize(RoundUp(
291        number_of_spill_slots * kVRegSize
292        + number_of_out_slots * kVRegSize
293        + maximum_number_of_live_core_registers * GetWordSize()
294        + maximum_number_of_live_fpu_registers * GetFloatingPointSpillSlotSize()
295        + FrameEntrySpillSize(),
296        kStackAlignment));
297  }
298}
299
300void CodeGenerator::CreateCommonInvokeLocationSummary(
301    HInvoke* invoke, InvokeDexCallingConventionVisitor* visitor) {
302  ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena();
303  LocationSummary* locations = new (allocator) LocationSummary(invoke, LocationSummary::kCall);
304
305  for (size_t i = 0; i < invoke->GetNumberOfArguments(); i++) {
306    HInstruction* input = invoke->InputAt(i);
307    locations->SetInAt(i, visitor->GetNextLocation(input->GetType()));
308  }
309
310  locations->SetOut(visitor->GetReturnLocation(invoke->GetType()));
311
312  if (invoke->IsInvokeStaticOrDirect()) {
313    HInvokeStaticOrDirect* call = invoke->AsInvokeStaticOrDirect();
314    switch (call->GetMethodLoadKind()) {
315      case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
316        locations->SetInAt(call->GetSpecialInputIndex(), visitor->GetMethodLocation());
317        break;
318      case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod:
319        locations->AddTemp(visitor->GetMethodLocation());
320        locations->SetInAt(call->GetSpecialInputIndex(), Location::RequiresRegister());
321        break;
322      default:
323        locations->AddTemp(visitor->GetMethodLocation());
324        break;
325    }
326  } else {
327    locations->AddTemp(visitor->GetMethodLocation());
328  }
329}
330
331void CodeGenerator::GenerateInvokeUnresolvedRuntimeCall(HInvokeUnresolved* invoke) {
332  MoveConstant(invoke->GetLocations()->GetTemp(0), invoke->GetDexMethodIndex());
333
334  // Initialize to anything to silent compiler warnings.
335  QuickEntrypointEnum entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
336  switch (invoke->GetOriginalInvokeType()) {
337    case kStatic:
338      entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
339      break;
340    case kDirect:
341      entrypoint = kQuickInvokeDirectTrampolineWithAccessCheck;
342      break;
343    case kVirtual:
344      entrypoint = kQuickInvokeVirtualTrampolineWithAccessCheck;
345      break;
346    case kSuper:
347      entrypoint = kQuickInvokeSuperTrampolineWithAccessCheck;
348      break;
349    case kInterface:
350      entrypoint = kQuickInvokeInterfaceTrampolineWithAccessCheck;
351      break;
352  }
353  InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), nullptr);
354}
355
356void CodeGenerator::CreateUnresolvedFieldLocationSummary(
357    HInstruction* field_access,
358    Primitive::Type field_type,
359    const FieldAccessCallingConvention& calling_convention) {
360  bool is_instance = field_access->IsUnresolvedInstanceFieldGet()
361      || field_access->IsUnresolvedInstanceFieldSet();
362  bool is_get = field_access->IsUnresolvedInstanceFieldGet()
363      || field_access->IsUnresolvedStaticFieldGet();
364
365  ArenaAllocator* allocator = field_access->GetBlock()->GetGraph()->GetArena();
366  LocationSummary* locations =
367      new (allocator) LocationSummary(field_access, LocationSummary::kCall);
368
369  locations->AddTemp(calling_convention.GetFieldIndexLocation());
370
371  if (is_instance) {
372    // Add the `this` object for instance field accesses.
373    locations->SetInAt(0, calling_convention.GetObjectLocation());
374  }
375
376  // Note that pSetXXStatic/pGetXXStatic always takes/returns an int or int64
377  // regardless of the the type. Because of that we forced to special case
378  // the access to floating point values.
379  if (is_get) {
380    if (Primitive::IsFloatingPointType(field_type)) {
381      // The return value will be stored in regular registers while register
382      // allocator expects it in a floating point register.
383      // Note We don't need to request additional temps because the return
384      // register(s) are already blocked due the call and they may overlap with
385      // the input or field index.
386      // The transfer between the two will be done at codegen level.
387      locations->SetOut(calling_convention.GetFpuLocation(field_type));
388    } else {
389      locations->SetOut(calling_convention.GetReturnLocation(field_type));
390    }
391  } else {
392     size_t set_index = is_instance ? 1 : 0;
393     if (Primitive::IsFloatingPointType(field_type)) {
394      // The set value comes from a float location while the calling convention
395      // expects it in a regular register location. Allocate a temp for it and
396      // make the transfer at codegen.
397      AddLocationAsTemp(calling_convention.GetSetValueLocation(field_type, is_instance), locations);
398      locations->SetInAt(set_index, calling_convention.GetFpuLocation(field_type));
399    } else {
400      locations->SetInAt(set_index,
401          calling_convention.GetSetValueLocation(field_type, is_instance));
402    }
403  }
404}
405
406void CodeGenerator::GenerateUnresolvedFieldAccess(
407    HInstruction* field_access,
408    Primitive::Type field_type,
409    uint32_t field_index,
410    uint32_t dex_pc,
411    const FieldAccessCallingConvention& calling_convention) {
412  LocationSummary* locations = field_access->GetLocations();
413
414  MoveConstant(locations->GetTemp(0), field_index);
415
416  bool is_instance = field_access->IsUnresolvedInstanceFieldGet()
417      || field_access->IsUnresolvedInstanceFieldSet();
418  bool is_get = field_access->IsUnresolvedInstanceFieldGet()
419      || field_access->IsUnresolvedStaticFieldGet();
420
421  if (!is_get && Primitive::IsFloatingPointType(field_type)) {
422    // Copy the float value to be set into the calling convention register.
423    // Note that using directly the temp location is problematic as we don't
424    // support temp register pairs. To avoid boilerplate conversion code, use
425    // the location from the calling convention.
426    MoveLocation(calling_convention.GetSetValueLocation(field_type, is_instance),
427                 locations->InAt(is_instance ? 1 : 0),
428                 (Primitive::Is64BitType(field_type) ? Primitive::kPrimLong : Primitive::kPrimInt));
429  }
430
431  QuickEntrypointEnum entrypoint = kQuickSet8Static;  // Initialize to anything to avoid warnings.
432  switch (field_type) {
433    case Primitive::kPrimBoolean:
434      entrypoint = is_instance
435          ? (is_get ? kQuickGetBooleanInstance : kQuickSet8Instance)
436          : (is_get ? kQuickGetBooleanStatic : kQuickSet8Static);
437      break;
438    case Primitive::kPrimByte:
439      entrypoint = is_instance
440          ? (is_get ? kQuickGetByteInstance : kQuickSet8Instance)
441          : (is_get ? kQuickGetByteStatic : kQuickSet8Static);
442      break;
443    case Primitive::kPrimShort:
444      entrypoint = is_instance
445          ? (is_get ? kQuickGetShortInstance : kQuickSet16Instance)
446          : (is_get ? kQuickGetShortStatic : kQuickSet16Static);
447      break;
448    case Primitive::kPrimChar:
449      entrypoint = is_instance
450          ? (is_get ? kQuickGetCharInstance : kQuickSet16Instance)
451          : (is_get ? kQuickGetCharStatic : kQuickSet16Static);
452      break;
453    case Primitive::kPrimInt:
454    case Primitive::kPrimFloat:
455      entrypoint = is_instance
456          ? (is_get ? kQuickGet32Instance : kQuickSet32Instance)
457          : (is_get ? kQuickGet32Static : kQuickSet32Static);
458      break;
459    case Primitive::kPrimNot:
460      entrypoint = is_instance
461          ? (is_get ? kQuickGetObjInstance : kQuickSetObjInstance)
462          : (is_get ? kQuickGetObjStatic : kQuickSetObjStatic);
463      break;
464    case Primitive::kPrimLong:
465    case Primitive::kPrimDouble:
466      entrypoint = is_instance
467          ? (is_get ? kQuickGet64Instance : kQuickSet64Instance)
468          : (is_get ? kQuickGet64Static : kQuickSet64Static);
469      break;
470    default:
471      LOG(FATAL) << "Invalid type " << field_type;
472  }
473  InvokeRuntime(entrypoint, field_access, dex_pc, nullptr);
474
475  if (is_get && Primitive::IsFloatingPointType(field_type)) {
476    MoveLocation(locations->Out(), calling_convention.GetReturnLocation(field_type), field_type);
477  }
478}
479
480// TODO: Remove argument `code_generator_supports_read_barrier` when
481// all code generators have read barrier support.
482void CodeGenerator::CreateLoadClassLocationSummary(HLoadClass* cls,
483                                                   Location runtime_type_index_location,
484                                                   Location runtime_return_location,
485                                                   bool code_generator_supports_read_barrier) {
486  ArenaAllocator* allocator = cls->GetBlock()->GetGraph()->GetArena();
487  LocationSummary::CallKind call_kind = cls->NeedsAccessCheck()
488      ? LocationSummary::kCall
489      : (((code_generator_supports_read_barrier && kEmitCompilerReadBarrier) ||
490          cls->CanCallRuntime())
491            ? LocationSummary::kCallOnSlowPath
492            : LocationSummary::kNoCall);
493  LocationSummary* locations = new (allocator) LocationSummary(cls, call_kind);
494  if (cls->NeedsAccessCheck()) {
495    locations->SetInAt(0, Location::NoLocation());
496    locations->AddTemp(runtime_type_index_location);
497    locations->SetOut(runtime_return_location);
498  } else {
499    locations->SetInAt(0, Location::RequiresRegister());
500    locations->SetOut(Location::RequiresRegister());
501  }
502}
503
504
505void CodeGenerator::BlockIfInRegister(Location location, bool is_out) const {
506  // The DCHECKS below check that a register is not specified twice in
507  // the summary. The out location can overlap with an input, so we need
508  // to special case it.
509  if (location.IsRegister()) {
510    DCHECK(is_out || !blocked_core_registers_[location.reg()]);
511    blocked_core_registers_[location.reg()] = true;
512  } else if (location.IsFpuRegister()) {
513    DCHECK(is_out || !blocked_fpu_registers_[location.reg()]);
514    blocked_fpu_registers_[location.reg()] = true;
515  } else if (location.IsFpuRegisterPair()) {
516    DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()]);
517    blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()] = true;
518    DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()]);
519    blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()] = true;
520  } else if (location.IsRegisterPair()) {
521    DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairLow<int>()]);
522    blocked_core_registers_[location.AsRegisterPairLow<int>()] = true;
523    DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairHigh<int>()]);
524    blocked_core_registers_[location.AsRegisterPairHigh<int>()] = true;
525  }
526}
527
528void CodeGenerator::AllocateLocations(HInstruction* instruction) {
529  instruction->Accept(GetLocationBuilder());
530  DCHECK(CheckTypeConsistency(instruction));
531  LocationSummary* locations = instruction->GetLocations();
532  if (!instruction->IsSuspendCheckEntry()) {
533    if (locations != nullptr) {
534      if (locations->CanCall()) {
535        MarkNotLeaf();
536      } else if (locations->Intrinsified() &&
537                 instruction->IsInvokeStaticOrDirect() &&
538                 !instruction->AsInvokeStaticOrDirect()->HasCurrentMethodInput()) {
539        // A static method call that has been fully intrinsified, and cannot call on the slow
540        // path or refer to the current method directly, no longer needs current method.
541        return;
542      }
543    }
544    if (instruction->NeedsCurrentMethod()) {
545      SetRequiresCurrentMethod();
546    }
547  }
548}
549
550void CodeGenerator::MaybeRecordStat(MethodCompilationStat compilation_stat, size_t count) const {
551  if (stats_ != nullptr) {
552    stats_->RecordStat(compilation_stat, count);
553  }
554}
555
556std::unique_ptr<CodeGenerator> CodeGenerator::Create(HGraph* graph,
557                                                     InstructionSet instruction_set,
558                                                     const InstructionSetFeatures& isa_features,
559                                                     const CompilerOptions& compiler_options,
560                                                     OptimizingCompilerStats* stats) {
561  ArenaAllocator* arena = graph->GetArena();
562  switch (instruction_set) {
563#ifdef ART_ENABLE_CODEGEN_arm
564    case kArm:
565    case kThumb2: {
566      return std::unique_ptr<CodeGenerator>(
567          new (arena) arm::CodeGeneratorARM(graph,
568                                            *isa_features.AsArmInstructionSetFeatures(),
569                                            compiler_options,
570                                            stats));
571    }
572#endif
573#ifdef ART_ENABLE_CODEGEN_arm64
574    case kArm64: {
575      return std::unique_ptr<CodeGenerator>(
576          new (arena) arm64::CodeGeneratorARM64(graph,
577                                                *isa_features.AsArm64InstructionSetFeatures(),
578                                                compiler_options,
579                                                stats));
580    }
581#endif
582#ifdef ART_ENABLE_CODEGEN_mips
583    case kMips: {
584      return std::unique_ptr<CodeGenerator>(
585          new (arena) mips::CodeGeneratorMIPS(graph,
586                                              *isa_features.AsMipsInstructionSetFeatures(),
587                                              compiler_options,
588                                              stats));
589    }
590#endif
591#ifdef ART_ENABLE_CODEGEN_mips64
592    case kMips64: {
593      return std::unique_ptr<CodeGenerator>(
594          new (arena) mips64::CodeGeneratorMIPS64(graph,
595                                                  *isa_features.AsMips64InstructionSetFeatures(),
596                                                  compiler_options,
597                                                  stats));
598    }
599#endif
600#ifdef ART_ENABLE_CODEGEN_x86
601    case kX86: {
602      return std::unique_ptr<CodeGenerator>(
603          new (arena) x86::CodeGeneratorX86(graph,
604                                            *isa_features.AsX86InstructionSetFeatures(),
605                                            compiler_options,
606                                            stats));
607    }
608#endif
609#ifdef ART_ENABLE_CODEGEN_x86_64
610    case kX86_64: {
611      return std::unique_ptr<CodeGenerator>(
612          new (arena) x86_64::CodeGeneratorX86_64(graph,
613                                                  *isa_features.AsX86_64InstructionSetFeatures(),
614                                                  compiler_options,
615                                                  stats));
616    }
617#endif
618    default:
619      return nullptr;
620  }
621}
622
623size_t CodeGenerator::ComputeStackMapsSize() {
624  return stack_map_stream_.PrepareForFillIn();
625}
626
627static void CheckCovers(uint32_t dex_pc,
628                        const HGraph& graph,
629                        const CodeInfo& code_info,
630                        const ArenaVector<HSuspendCheck*>& loop_headers,
631                        ArenaVector<size_t>* covered) {
632  CodeInfoEncoding encoding = code_info.ExtractEncoding();
633  for (size_t i = 0; i < loop_headers.size(); ++i) {
634    if (loop_headers[i]->GetDexPc() == dex_pc) {
635      if (graph.IsCompilingOsr()) {
636        DCHECK(code_info.GetOsrStackMapForDexPc(dex_pc, encoding).IsValid());
637      }
638      ++(*covered)[i];
639    }
640  }
641}
642
643// Debug helper to ensure loop entries in compiled code are matched by
644// dex branch instructions.
645static void CheckLoopEntriesCanBeUsedForOsr(const HGraph& graph,
646                                            const CodeInfo& code_info,
647                                            const DexFile::CodeItem& code_item) {
648  if (graph.HasTryCatch()) {
649    // One can write loops through try/catch, which we do not support for OSR anyway.
650    return;
651  }
652  ArenaVector<HSuspendCheck*> loop_headers(graph.GetArena()->Adapter(kArenaAllocMisc));
653  for (HReversePostOrderIterator it(graph); !it.Done(); it.Advance()) {
654    if (it.Current()->IsLoopHeader()) {
655      HSuspendCheck* suspend_check = it.Current()->GetLoopInformation()->GetSuspendCheck();
656      if (!suspend_check->GetEnvironment()->IsFromInlinedInvoke()) {
657        loop_headers.push_back(suspend_check);
658      }
659    }
660  }
661  ArenaVector<size_t> covered(loop_headers.size(), 0, graph.GetArena()->Adapter(kArenaAllocMisc));
662  const uint16_t* code_ptr = code_item.insns_;
663  const uint16_t* code_end = code_item.insns_ + code_item.insns_size_in_code_units_;
664
665  size_t dex_pc = 0;
666  while (code_ptr < code_end) {
667    const Instruction& instruction = *Instruction::At(code_ptr);
668    if (instruction.IsBranch()) {
669      uint32_t target = dex_pc + instruction.GetTargetOffset();
670      CheckCovers(target, graph, code_info, loop_headers, &covered);
671    } else if (instruction.IsSwitch()) {
672      DexSwitchTable table(instruction, dex_pc);
673      uint16_t num_entries = table.GetNumEntries();
674      size_t offset = table.GetFirstValueIndex();
675
676      // Use a larger loop counter type to avoid overflow issues.
677      for (size_t i = 0; i < num_entries; ++i) {
678        // The target of the case.
679        uint32_t target = dex_pc + table.GetEntryAt(i + offset);
680        CheckCovers(target, graph, code_info, loop_headers, &covered);
681      }
682    }
683    dex_pc += instruction.SizeInCodeUnits();
684    code_ptr += instruction.SizeInCodeUnits();
685  }
686
687  for (size_t i = 0; i < covered.size(); ++i) {
688    DCHECK_NE(covered[i], 0u) << "Loop in compiled code has no dex branch equivalent";
689  }
690}
691
692void CodeGenerator::BuildStackMaps(MemoryRegion region, const DexFile::CodeItem& code_item) {
693  stack_map_stream_.FillIn(region);
694  if (kIsDebugBuild) {
695    CheckLoopEntriesCanBeUsedForOsr(*graph_, CodeInfo(region), code_item);
696  }
697}
698
699void CodeGenerator::RecordPcInfo(HInstruction* instruction,
700                                 uint32_t dex_pc,
701                                 SlowPathCode* slow_path) {
702  if (instruction != nullptr) {
703    // The code generated for some type conversions
704    // may call the runtime, thus normally requiring a subsequent
705    // call to this method. However, the method verifier does not
706    // produce PC information for certain instructions, which are
707    // considered "atomic" (they cannot join a GC).
708    // Therefore we do not currently record PC information for such
709    // instructions.  As this may change later, we added this special
710    // case so that code generators may nevertheless call
711    // CodeGenerator::RecordPcInfo without triggering an error in
712    // CodeGenerator::BuildNativeGCMap ("Missing ref for dex pc 0x")
713    // thereafter.
714    if (instruction->IsTypeConversion()) {
715      return;
716    }
717    if (instruction->IsRem()) {
718      Primitive::Type type = instruction->AsRem()->GetResultType();
719      if ((type == Primitive::kPrimFloat) || (type == Primitive::kPrimDouble)) {
720        return;
721      }
722    }
723  }
724
725  uint32_t outer_dex_pc = dex_pc;
726  uint32_t outer_environment_size = 0;
727  uint32_t inlining_depth = 0;
728  if (instruction != nullptr) {
729    for (HEnvironment* environment = instruction->GetEnvironment();
730         environment != nullptr;
731         environment = environment->GetParent()) {
732      outer_dex_pc = environment->GetDexPc();
733      outer_environment_size = environment->Size();
734      if (environment != instruction->GetEnvironment()) {
735        inlining_depth++;
736      }
737    }
738  }
739
740  // Collect PC infos for the mapping table.
741  uint32_t native_pc = GetAssembler()->CodeSize();
742
743  if (instruction == nullptr) {
744    // For stack overflow checks and native-debug-info entries without dex register
745    // mapping (i.e. start of basic block or start of slow path).
746    stack_map_stream_.BeginStackMapEntry(outer_dex_pc, native_pc, 0, 0, 0, 0);
747    stack_map_stream_.EndStackMapEntry();
748    return;
749  }
750  LocationSummary* locations = instruction->GetLocations();
751
752  uint32_t register_mask = locations->GetRegisterMask();
753  if (locations->OnlyCallsOnSlowPath()) {
754    // In case of slow path, we currently set the location of caller-save registers
755    // to register (instead of their stack location when pushed before the slow-path
756    // call). Therefore register_mask contains both callee-save and caller-save
757    // registers that hold objects. We must remove the caller-save from the mask, since
758    // they will be overwritten by the callee.
759    register_mask &= core_callee_save_mask_;
760  }
761  // The register mask must be a subset of callee-save registers.
762  DCHECK_EQ(register_mask & core_callee_save_mask_, register_mask);
763  stack_map_stream_.BeginStackMapEntry(outer_dex_pc,
764                                       native_pc,
765                                       register_mask,
766                                       locations->GetStackMask(),
767                                       outer_environment_size,
768                                       inlining_depth);
769
770  EmitEnvironment(instruction->GetEnvironment(), slow_path);
771  stack_map_stream_.EndStackMapEntry();
772
773  HLoopInformation* info = instruction->GetBlock()->GetLoopInformation();
774  if (instruction->IsSuspendCheck() &&
775      (info != nullptr) &&
776      graph_->IsCompilingOsr() &&
777      (inlining_depth == 0)) {
778    DCHECK_EQ(info->GetSuspendCheck(), instruction);
779    // We duplicate the stack map as a marker that this stack map can be an OSR entry.
780    // Duplicating it avoids having the runtime recognize and skip an OSR stack map.
781    DCHECK(info->IsIrreducible());
782    stack_map_stream_.BeginStackMapEntry(
783        dex_pc, native_pc, register_mask, locations->GetStackMask(), outer_environment_size, 0);
784    EmitEnvironment(instruction->GetEnvironment(), slow_path);
785    stack_map_stream_.EndStackMapEntry();
786    if (kIsDebugBuild) {
787      HEnvironment* environment = instruction->GetEnvironment();
788      for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) {
789        HInstruction* in_environment = environment->GetInstructionAt(i);
790        if (in_environment != nullptr) {
791          DCHECK(in_environment->IsPhi() || in_environment->IsConstant());
792          Location location = environment->GetLocationAt(i);
793          DCHECK(location.IsStackSlot() ||
794                 location.IsDoubleStackSlot() ||
795                 location.IsConstant() ||
796                 location.IsInvalid());
797          if (location.IsStackSlot() || location.IsDoubleStackSlot()) {
798            DCHECK_LT(location.GetStackIndex(), static_cast<int32_t>(GetFrameSize()));
799          }
800        }
801      }
802    }
803  } else if (kIsDebugBuild) {
804    // Ensure stack maps are unique, by checking that the native pc in the stack map
805    // last emitted is different than the native pc of the stack map just emitted.
806    size_t number_of_stack_maps = stack_map_stream_.GetNumberOfStackMaps();
807    if (number_of_stack_maps > 1) {
808      DCHECK_NE(stack_map_stream_.GetStackMap(number_of_stack_maps - 1).native_pc_offset,
809                stack_map_stream_.GetStackMap(number_of_stack_maps - 2).native_pc_offset);
810    }
811  }
812}
813
814bool CodeGenerator::HasStackMapAtCurrentPc() {
815  uint32_t pc = GetAssembler()->CodeSize();
816  size_t count = stack_map_stream_.GetNumberOfStackMaps();
817  return count > 0 && stack_map_stream_.GetStackMap(count - 1).native_pc_offset == pc;
818}
819
820void CodeGenerator::MaybeRecordNativeDebugInfo(HInstruction* instruction,
821                                               uint32_t dex_pc,
822                                               SlowPathCode* slow_path) {
823  if (GetCompilerOptions().GetNativeDebuggable() && dex_pc != kNoDexPc) {
824    if (HasStackMapAtCurrentPc()) {
825      // Ensure that we do not collide with the stack map of the previous instruction.
826      GenerateNop();
827    }
828    RecordPcInfo(instruction, dex_pc, slow_path);
829  }
830}
831
832void CodeGenerator::RecordCatchBlockInfo() {
833  ArenaAllocator* arena = graph_->GetArena();
834
835  for (HBasicBlock* block : *block_order_) {
836    if (!block->IsCatchBlock()) {
837      continue;
838    }
839
840    uint32_t dex_pc = block->GetDexPc();
841    uint32_t num_vregs = graph_->GetNumberOfVRegs();
842    uint32_t inlining_depth = 0;  // Inlining of catch blocks is not supported at the moment.
843    uint32_t native_pc = GetAddressOf(block);
844    uint32_t register_mask = 0;   // Not used.
845
846    // The stack mask is not used, so we leave it empty.
847    ArenaBitVector* stack_mask =
848        ArenaBitVector::Create(arena, 0, /* expandable */ true, kArenaAllocCodeGenerator);
849
850    stack_map_stream_.BeginStackMapEntry(dex_pc,
851                                         native_pc,
852                                         register_mask,
853                                         stack_mask,
854                                         num_vregs,
855                                         inlining_depth);
856
857    HInstruction* current_phi = block->GetFirstPhi();
858    for (size_t vreg = 0; vreg < num_vregs; ++vreg) {
859    while (current_phi != nullptr && current_phi->AsPhi()->GetRegNumber() < vreg) {
860      HInstruction* next_phi = current_phi->GetNext();
861      DCHECK(next_phi == nullptr ||
862             current_phi->AsPhi()->GetRegNumber() <= next_phi->AsPhi()->GetRegNumber())
863          << "Phis need to be sorted by vreg number to keep this a linear-time loop.";
864      current_phi = next_phi;
865    }
866
867      if (current_phi == nullptr || current_phi->AsPhi()->GetRegNumber() != vreg) {
868        stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
869      } else {
870        Location location = current_phi->GetLiveInterval()->ToLocation();
871        switch (location.GetKind()) {
872          case Location::kStackSlot: {
873            stack_map_stream_.AddDexRegisterEntry(
874                DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
875            break;
876          }
877          case Location::kDoubleStackSlot: {
878            stack_map_stream_.AddDexRegisterEntry(
879                DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
880            stack_map_stream_.AddDexRegisterEntry(
881                DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize));
882            ++vreg;
883            DCHECK_LT(vreg, num_vregs);
884            break;
885          }
886          default: {
887            // All catch phis must be allocated to a stack slot.
888            LOG(FATAL) << "Unexpected kind " << location.GetKind();
889            UNREACHABLE();
890          }
891        }
892      }
893    }
894
895    stack_map_stream_.EndStackMapEntry();
896  }
897}
898
899void CodeGenerator::EmitEnvironment(HEnvironment* environment, SlowPathCode* slow_path) {
900  if (environment == nullptr) return;
901
902  if (environment->GetParent() != nullptr) {
903    // We emit the parent environment first.
904    EmitEnvironment(environment->GetParent(), slow_path);
905    stack_map_stream_.BeginInlineInfoEntry(environment->GetMethodIdx(),
906                                           environment->GetDexPc(),
907                                           environment->GetInvokeType(),
908                                           environment->Size());
909  }
910
911  // Walk over the environment, and record the location of dex registers.
912  for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) {
913    HInstruction* current = environment->GetInstructionAt(i);
914    if (current == nullptr) {
915      stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
916      continue;
917    }
918
919    Location location = environment->GetLocationAt(i);
920    switch (location.GetKind()) {
921      case Location::kConstant: {
922        DCHECK_EQ(current, location.GetConstant());
923        if (current->IsLongConstant()) {
924          int64_t value = current->AsLongConstant()->GetValue();
925          stack_map_stream_.AddDexRegisterEntry(
926              DexRegisterLocation::Kind::kConstant, Low32Bits(value));
927          stack_map_stream_.AddDexRegisterEntry(
928              DexRegisterLocation::Kind::kConstant, High32Bits(value));
929          ++i;
930          DCHECK_LT(i, environment_size);
931        } else if (current->IsDoubleConstant()) {
932          int64_t value = bit_cast<int64_t, double>(current->AsDoubleConstant()->GetValue());
933          stack_map_stream_.AddDexRegisterEntry(
934              DexRegisterLocation::Kind::kConstant, Low32Bits(value));
935          stack_map_stream_.AddDexRegisterEntry(
936              DexRegisterLocation::Kind::kConstant, High32Bits(value));
937          ++i;
938          DCHECK_LT(i, environment_size);
939        } else if (current->IsIntConstant()) {
940          int32_t value = current->AsIntConstant()->GetValue();
941          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, value);
942        } else if (current->IsNullConstant()) {
943          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, 0);
944        } else {
945          DCHECK(current->IsFloatConstant()) << current->DebugName();
946          int32_t value = bit_cast<int32_t, float>(current->AsFloatConstant()->GetValue());
947          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, value);
948        }
949        break;
950      }
951
952      case Location::kStackSlot: {
953        stack_map_stream_.AddDexRegisterEntry(
954            DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
955        break;
956      }
957
958      case Location::kDoubleStackSlot: {
959        stack_map_stream_.AddDexRegisterEntry(
960            DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
961        stack_map_stream_.AddDexRegisterEntry(
962            DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize));
963        ++i;
964        DCHECK_LT(i, environment_size);
965        break;
966      }
967
968      case Location::kRegister : {
969        int id = location.reg();
970        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(id)) {
971          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(id);
972          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
973          if (current->GetType() == Primitive::kPrimLong) {
974            stack_map_stream_.AddDexRegisterEntry(
975                DexRegisterLocation::Kind::kInStack, offset + kVRegSize);
976            ++i;
977            DCHECK_LT(i, environment_size);
978          }
979        } else {
980          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, id);
981          if (current->GetType() == Primitive::kPrimLong) {
982            stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegisterHigh, id);
983            ++i;
984            DCHECK_LT(i, environment_size);
985          }
986        }
987        break;
988      }
989
990      case Location::kFpuRegister : {
991        int id = location.reg();
992        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(id)) {
993          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(id);
994          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
995          if (current->GetType() == Primitive::kPrimDouble) {
996            stack_map_stream_.AddDexRegisterEntry(
997                DexRegisterLocation::Kind::kInStack, offset + kVRegSize);
998            ++i;
999            DCHECK_LT(i, environment_size);
1000          }
1001        } else {
1002          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, id);
1003          if (current->GetType() == Primitive::kPrimDouble) {
1004            stack_map_stream_.AddDexRegisterEntry(
1005                DexRegisterLocation::Kind::kInFpuRegisterHigh, id);
1006            ++i;
1007            DCHECK_LT(i, environment_size);
1008          }
1009        }
1010        break;
1011      }
1012
1013      case Location::kFpuRegisterPair : {
1014        int low = location.low();
1015        int high = location.high();
1016        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(low)) {
1017          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(low);
1018          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1019        } else {
1020          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, low);
1021        }
1022        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(high)) {
1023          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(high);
1024          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1025          ++i;
1026        } else {
1027          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, high);
1028          ++i;
1029        }
1030        DCHECK_LT(i, environment_size);
1031        break;
1032      }
1033
1034      case Location::kRegisterPair : {
1035        int low = location.low();
1036        int high = location.high();
1037        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(low)) {
1038          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(low);
1039          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1040        } else {
1041          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, low);
1042        }
1043        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(high)) {
1044          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(high);
1045          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1046        } else {
1047          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, high);
1048        }
1049        ++i;
1050        DCHECK_LT(i, environment_size);
1051        break;
1052      }
1053
1054      case Location::kInvalid: {
1055        stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
1056        break;
1057      }
1058
1059      default:
1060        LOG(FATAL) << "Unexpected kind " << location.GetKind();
1061    }
1062  }
1063
1064  if (environment->GetParent() != nullptr) {
1065    stack_map_stream_.EndInlineInfoEntry();
1066  }
1067}
1068
1069bool CodeGenerator::IsImplicitNullCheckAllowed(HNullCheck* null_check) const {
1070  return compiler_options_.GetImplicitNullChecks() &&
1071         // Null checks which might throw into a catch block need to save live
1072         // registers and therefore cannot be done implicitly.
1073         !null_check->CanThrowIntoCatchBlock();
1074}
1075
1076bool CodeGenerator::CanMoveNullCheckToUser(HNullCheck* null_check) {
1077  HInstruction* first_next_not_move = null_check->GetNextDisregardingMoves();
1078
1079  return (first_next_not_move != nullptr)
1080      && first_next_not_move->CanDoImplicitNullCheckOn(null_check->InputAt(0));
1081}
1082
1083void CodeGenerator::MaybeRecordImplicitNullCheck(HInstruction* instr) {
1084  // If we are from a static path don't record the pc as we can't throw NPE.
1085  // NB: having the checks here makes the code much less verbose in the arch
1086  // specific code generators.
1087  if (instr->IsStaticFieldSet() || instr->IsStaticFieldGet()) {
1088    return;
1089  }
1090
1091  if (!instr->CanDoImplicitNullCheckOn(instr->InputAt(0))) {
1092    return;
1093  }
1094
1095  // Find the first previous instruction which is not a move.
1096  HInstruction* first_prev_not_move = instr->GetPreviousDisregardingMoves();
1097
1098  // If the instruction is a null check it means that `instr` is the first user
1099  // and needs to record the pc.
1100  if (first_prev_not_move != nullptr && first_prev_not_move->IsNullCheck()) {
1101    HNullCheck* null_check = first_prev_not_move->AsNullCheck();
1102    if (IsImplicitNullCheckAllowed(null_check)) {
1103      // TODO: The parallel moves modify the environment. Their changes need to be
1104      // reverted otherwise the stack maps at the throw point will not be correct.
1105      RecordPcInfo(null_check, null_check->GetDexPc());
1106    }
1107  }
1108}
1109
1110void CodeGenerator::GenerateNullCheck(HNullCheck* instruction) {
1111  if (IsImplicitNullCheckAllowed(instruction)) {
1112    MaybeRecordStat(kImplicitNullCheckGenerated);
1113    GenerateImplicitNullCheck(instruction);
1114  } else {
1115    MaybeRecordStat(kExplicitNullCheckGenerated);
1116    GenerateExplicitNullCheck(instruction);
1117  }
1118}
1119
1120void CodeGenerator::ClearSpillSlotsFromLoopPhisInStackMap(HSuspendCheck* suspend_check) const {
1121  LocationSummary* locations = suspend_check->GetLocations();
1122  HBasicBlock* block = suspend_check->GetBlock();
1123  DCHECK(block->GetLoopInformation()->GetSuspendCheck() == suspend_check);
1124  DCHECK(block->IsLoopHeader());
1125
1126  for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) {
1127    HInstruction* current = it.Current();
1128    LiveInterval* interval = current->GetLiveInterval();
1129    // We only need to clear bits of loop phis containing objects and allocated in register.
1130    // Loop phis allocated on stack already have the object in the stack.
1131    if (current->GetType() == Primitive::kPrimNot
1132        && interval->HasRegister()
1133        && interval->HasSpillSlot()) {
1134      locations->ClearStackBit(interval->GetSpillSlot() / kVRegSize);
1135    }
1136  }
1137}
1138
1139void CodeGenerator::EmitParallelMoves(Location from1,
1140                                      Location to1,
1141                                      Primitive::Type type1,
1142                                      Location from2,
1143                                      Location to2,
1144                                      Primitive::Type type2) {
1145  HParallelMove parallel_move(GetGraph()->GetArena());
1146  parallel_move.AddMove(from1, to1, type1, nullptr);
1147  parallel_move.AddMove(from2, to2, type2, nullptr);
1148  GetMoveResolver()->EmitNativeCode(&parallel_move);
1149}
1150
1151void CodeGenerator::ValidateInvokeRuntime(HInstruction* instruction, SlowPathCode* slow_path) {
1152  // Ensure that the call kind indication given to the register allocator is
1153  // coherent with the runtime call generated, and that the GC side effect is
1154  // set when required.
1155  if (slow_path == nullptr) {
1156    DCHECK(instruction->GetLocations()->WillCall())
1157        << "instruction->DebugName()=" << instruction->DebugName();
1158    DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()))
1159        << "instruction->DebugName()=" << instruction->DebugName()
1160        << " instruction->GetSideEffects().ToString()=" << instruction->GetSideEffects().ToString();
1161  } else {
1162    DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath() || slow_path->IsFatal())
1163        << "instruction->DebugName()=" << instruction->DebugName()
1164        << " slow_path->GetDescription()=" << slow_path->GetDescription();
1165    DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()) ||
1166           // When read barriers are enabled, some instructions use a
1167           // slow path to emit a read barrier, which does not trigger
1168           // GC, is not fatal, nor is emitted by HDeoptimize
1169           // instructions.
1170           (kEmitCompilerReadBarrier &&
1171            (instruction->IsInstanceFieldGet() ||
1172             instruction->IsStaticFieldGet() ||
1173             instruction->IsArraySet() ||
1174             instruction->IsArrayGet() ||
1175             instruction->IsLoadClass() ||
1176             instruction->IsLoadString() ||
1177             instruction->IsInstanceOf() ||
1178             instruction->IsCheckCast())))
1179        << "instruction->DebugName()=" << instruction->DebugName()
1180        << " instruction->GetSideEffects().ToString()=" << instruction->GetSideEffects().ToString()
1181        << " slow_path->GetDescription()=" << slow_path->GetDescription();
1182  }
1183
1184  // Check the coherency of leaf information.
1185  DCHECK(instruction->IsSuspendCheck()
1186         || ((slow_path != nullptr) && slow_path->IsFatal())
1187         || instruction->GetLocations()->CanCall()
1188         || !IsLeafMethod())
1189      << instruction->DebugName() << ((slow_path != nullptr) ? slow_path->GetDescription() : "");
1190}
1191
1192void SlowPathCode::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
1193  RegisterSet* live_registers = locations->GetLiveRegisters();
1194  size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
1195
1196  for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
1197    if (!codegen->IsCoreCalleeSaveRegister(i)) {
1198      if (live_registers->ContainsCoreRegister(i)) {
1199        // If the register holds an object, update the stack mask.
1200        if (locations->RegisterContainsObject(i)) {
1201          locations->SetStackBit(stack_offset / kVRegSize);
1202        }
1203        DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1204        DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1205        saved_core_stack_offsets_[i] = stack_offset;
1206        stack_offset += codegen->SaveCoreRegister(stack_offset, i);
1207      }
1208    }
1209  }
1210
1211  for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) {
1212    if (!codegen->IsFloatingPointCalleeSaveRegister(i)) {
1213      if (live_registers->ContainsFloatingPointRegister(i)) {
1214        DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1215        DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1216        saved_fpu_stack_offsets_[i] = stack_offset;
1217        stack_offset += codegen->SaveFloatingPointRegister(stack_offset, i);
1218      }
1219    }
1220  }
1221}
1222
1223void SlowPathCode::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
1224  RegisterSet* live_registers = locations->GetLiveRegisters();
1225  size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
1226
1227  for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
1228    if (!codegen->IsCoreCalleeSaveRegister(i)) {
1229      if (live_registers->ContainsCoreRegister(i)) {
1230        DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1231        DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1232        stack_offset += codegen->RestoreCoreRegister(stack_offset, i);
1233      }
1234    }
1235  }
1236
1237  for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) {
1238    if (!codegen->IsFloatingPointCalleeSaveRegister(i)) {
1239      if (live_registers->ContainsFloatingPointRegister(i)) {
1240        DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1241        DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1242        stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, i);
1243      }
1244    }
1245  }
1246}
1247
1248void CodeGenerator::CreateSystemArrayCopyLocationSummary(HInvoke* invoke) {
1249  // Check to see if we have known failures that will cause us to have to bail out
1250  // to the runtime, and just generate the runtime call directly.
1251  HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant();
1252  HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant();
1253
1254  // The positions must be non-negative.
1255  if ((src_pos != nullptr && src_pos->GetValue() < 0) ||
1256      (dest_pos != nullptr && dest_pos->GetValue() < 0)) {
1257    // We will have to fail anyways.
1258    return;
1259  }
1260
1261  // The length must be >= 0.
1262  HIntConstant* length = invoke->InputAt(4)->AsIntConstant();
1263  if (length != nullptr) {
1264    int32_t len = length->GetValue();
1265    if (len < 0) {
1266      // Just call as normal.
1267      return;
1268    }
1269  }
1270
1271  SystemArrayCopyOptimizations optimizations(invoke);
1272
1273  if (optimizations.GetDestinationIsSource()) {
1274    if (src_pos != nullptr && dest_pos != nullptr && src_pos->GetValue() < dest_pos->GetValue()) {
1275      // We only support backward copying if source and destination are the same.
1276      return;
1277    }
1278  }
1279
1280  if (optimizations.GetDestinationIsPrimitiveArray() || optimizations.GetSourceIsPrimitiveArray()) {
1281    // We currently don't intrinsify primitive copying.
1282    return;
1283  }
1284
1285  ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena();
1286  LocationSummary* locations = new (allocator) LocationSummary(invoke,
1287                                                               LocationSummary::kCallOnSlowPath,
1288                                                               kIntrinsified);
1289  // arraycopy(Object src, int src_pos, Object dest, int dest_pos, int length).
1290  locations->SetInAt(0, Location::RequiresRegister());
1291  locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1)));
1292  locations->SetInAt(2, Location::RequiresRegister());
1293  locations->SetInAt(3, Location::RegisterOrConstant(invoke->InputAt(3)));
1294  locations->SetInAt(4, Location::RegisterOrConstant(invoke->InputAt(4)));
1295
1296  locations->AddTemp(Location::RequiresRegister());
1297  locations->AddTemp(Location::RequiresRegister());
1298  locations->AddTemp(Location::RequiresRegister());
1299}
1300
1301}  // namespace art
1302