code_generator.cc revision 4acd03638fcdb4e5d1666f8eec7eb3bf6d6be035 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator.h"
18
19#ifdef ART_ENABLE_CODEGEN_arm
20#include "code_generator_arm.h"
21#include "code_generator_arm_vixl.h"
22#endif
23
24#ifdef ART_ENABLE_CODEGEN_arm64
25#include "code_generator_arm64.h"
26#endif
27
28#ifdef ART_ENABLE_CODEGEN_x86
29#include "code_generator_x86.h"
30#endif
31
32#ifdef ART_ENABLE_CODEGEN_x86_64
33#include "code_generator_x86_64.h"
34#endif
35
36#ifdef ART_ENABLE_CODEGEN_mips
37#include "code_generator_mips.h"
38#endif
39
40#ifdef ART_ENABLE_CODEGEN_mips64
41#include "code_generator_mips64.h"
42#endif
43
44#include "bytecode_utils.h"
45#include "compiled_method.h"
46#include "dex/verified_method.h"
47#include "driver/compiler_driver.h"
48#include "graph_visualizer.h"
49#include "intrinsics.h"
50#include "leb128.h"
51#include "mirror/array-inl.h"
52#include "mirror/object_array-inl.h"
53#include "mirror/object_reference.h"
54#include "mirror/string.h"
55#include "parallel_move_resolver.h"
56#include "ssa_liveness_analysis.h"
57#include "utils/assembler.h"
58
59namespace art {
60
61// Return whether a location is consistent with a type.
62static bool CheckType(Primitive::Type type, Location location) {
63  if (location.IsFpuRegister()
64      || (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresFpuRegister))) {
65    return (type == Primitive::kPrimFloat) || (type == Primitive::kPrimDouble);
66  } else if (location.IsRegister() ||
67             (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresRegister))) {
68    return Primitive::IsIntegralType(type) || (type == Primitive::kPrimNot);
69  } else if (location.IsRegisterPair()) {
70    return type == Primitive::kPrimLong;
71  } else if (location.IsFpuRegisterPair()) {
72    return type == Primitive::kPrimDouble;
73  } else if (location.IsStackSlot()) {
74    return (Primitive::IsIntegralType(type) && type != Primitive::kPrimLong)
75           || (type == Primitive::kPrimFloat)
76           || (type == Primitive::kPrimNot);
77  } else if (location.IsDoubleStackSlot()) {
78    return (type == Primitive::kPrimLong) || (type == Primitive::kPrimDouble);
79  } else if (location.IsConstant()) {
80    if (location.GetConstant()->IsIntConstant()) {
81      return Primitive::IsIntegralType(type) && (type != Primitive::kPrimLong);
82    } else if (location.GetConstant()->IsNullConstant()) {
83      return type == Primitive::kPrimNot;
84    } else if (location.GetConstant()->IsLongConstant()) {
85      return type == Primitive::kPrimLong;
86    } else if (location.GetConstant()->IsFloatConstant()) {
87      return type == Primitive::kPrimFloat;
88    } else {
89      return location.GetConstant()->IsDoubleConstant()
90          && (type == Primitive::kPrimDouble);
91    }
92  } else {
93    return location.IsInvalid() || (location.GetPolicy() == Location::kAny);
94  }
95}
96
97// Check that a location summary is consistent with an instruction.
98static bool CheckTypeConsistency(HInstruction* instruction) {
99  LocationSummary* locations = instruction->GetLocations();
100  if (locations == nullptr) {
101    return true;
102  }
103
104  if (locations->Out().IsUnallocated()
105      && (locations->Out().GetPolicy() == Location::kSameAsFirstInput)) {
106    DCHECK(CheckType(instruction->GetType(), locations->InAt(0)))
107        << instruction->GetType()
108        << " " << locations->InAt(0);
109  } else {
110    DCHECK(CheckType(instruction->GetType(), locations->Out()))
111        << instruction->GetType()
112        << " " << locations->Out();
113  }
114
115  HConstInputsRef inputs = instruction->GetInputs();
116  for (size_t i = 0; i < inputs.size(); ++i) {
117    DCHECK(CheckType(inputs[i]->GetType(), locations->InAt(i)))
118      << inputs[i]->GetType() << " " << locations->InAt(i);
119  }
120
121  HEnvironment* environment = instruction->GetEnvironment();
122  for (size_t i = 0; i < instruction->EnvironmentSize(); ++i) {
123    if (environment->GetInstructionAt(i) != nullptr) {
124      Primitive::Type type = environment->GetInstructionAt(i)->GetType();
125      DCHECK(CheckType(type, environment->GetLocationAt(i)))
126        << type << " " << environment->GetLocationAt(i);
127    } else {
128      DCHECK(environment->GetLocationAt(i).IsInvalid())
129        << environment->GetLocationAt(i);
130    }
131  }
132  return true;
133}
134
135size_t CodeGenerator::GetCacheOffset(uint32_t index) {
136  return sizeof(GcRoot<mirror::Object>) * index;
137}
138
139size_t CodeGenerator::GetCachePointerOffset(uint32_t index) {
140  auto pointer_size = InstructionSetPointerSize(GetInstructionSet());
141  return static_cast<size_t>(pointer_size) * index;
142}
143
144uint32_t CodeGenerator::GetArrayLengthOffset(HArrayLength* array_length) {
145  return array_length->IsStringLength()
146      ? mirror::String::CountOffset().Uint32Value()
147      : mirror::Array::LengthOffset().Uint32Value();
148}
149
150uint32_t CodeGenerator::GetArrayDataOffset(HArrayGet* array_get) {
151  DCHECK(array_get->GetType() == Primitive::kPrimChar || !array_get->IsStringCharAt());
152  return array_get->IsStringCharAt()
153      ? mirror::String::ValueOffset().Uint32Value()
154      : mirror::Array::DataOffset(Primitive::ComponentSize(array_get->GetType())).Uint32Value();
155}
156
157bool CodeGenerator::GoesToNextBlock(HBasicBlock* current, HBasicBlock* next) const {
158  DCHECK_EQ((*block_order_)[current_block_index_], current);
159  return GetNextBlockToEmit() == FirstNonEmptyBlock(next);
160}
161
162HBasicBlock* CodeGenerator::GetNextBlockToEmit() const {
163  for (size_t i = current_block_index_ + 1; i < block_order_->size(); ++i) {
164    HBasicBlock* block = (*block_order_)[i];
165    if (!block->IsSingleJump()) {
166      return block;
167    }
168  }
169  return nullptr;
170}
171
172HBasicBlock* CodeGenerator::FirstNonEmptyBlock(HBasicBlock* block) const {
173  while (block->IsSingleJump()) {
174    block = block->GetSuccessors()[0];
175  }
176  return block;
177}
178
179class DisassemblyScope {
180 public:
181  DisassemblyScope(HInstruction* instruction, const CodeGenerator& codegen)
182      : codegen_(codegen), instruction_(instruction), start_offset_(static_cast<size_t>(-1)) {
183    if (codegen_.GetDisassemblyInformation() != nullptr) {
184      start_offset_ = codegen_.GetAssembler().CodeSize();
185    }
186  }
187
188  ~DisassemblyScope() {
189    // We avoid building this data when we know it will not be used.
190    if (codegen_.GetDisassemblyInformation() != nullptr) {
191      codegen_.GetDisassemblyInformation()->AddInstructionInterval(
192          instruction_, start_offset_, codegen_.GetAssembler().CodeSize());
193    }
194  }
195
196 private:
197  const CodeGenerator& codegen_;
198  HInstruction* instruction_;
199  size_t start_offset_;
200};
201
202
203void CodeGenerator::GenerateSlowPaths() {
204  size_t code_start = 0;
205  for (const std::unique_ptr<SlowPathCode>& slow_path_unique_ptr : slow_paths_) {
206    SlowPathCode* slow_path = slow_path_unique_ptr.get();
207    current_slow_path_ = slow_path;
208    if (disasm_info_ != nullptr) {
209      code_start = GetAssembler()->CodeSize();
210    }
211    // Record the dex pc at start of slow path (required for java line number mapping).
212    MaybeRecordNativeDebugInfo(slow_path->GetInstruction(), slow_path->GetDexPc(), slow_path);
213    slow_path->EmitNativeCode(this);
214    if (disasm_info_ != nullptr) {
215      disasm_info_->AddSlowPathInterval(slow_path, code_start, GetAssembler()->CodeSize());
216    }
217  }
218  current_slow_path_ = nullptr;
219}
220
221void CodeGenerator::Compile(CodeAllocator* allocator) {
222  // The register allocator already called `InitializeCodeGeneration`,
223  // where the frame size has been computed.
224  DCHECK(block_order_ != nullptr);
225  Initialize();
226
227  HGraphVisitor* instruction_visitor = GetInstructionVisitor();
228  DCHECK_EQ(current_block_index_, 0u);
229
230  size_t frame_start = GetAssembler()->CodeSize();
231  GenerateFrameEntry();
232  DCHECK_EQ(GetAssembler()->cfi().GetCurrentCFAOffset(), static_cast<int>(frame_size_));
233  if (disasm_info_ != nullptr) {
234    disasm_info_->SetFrameEntryInterval(frame_start, GetAssembler()->CodeSize());
235  }
236
237  for (size_t e = block_order_->size(); current_block_index_ < e; ++current_block_index_) {
238    HBasicBlock* block = (*block_order_)[current_block_index_];
239    // Don't generate code for an empty block. Its predecessors will branch to its successor
240    // directly. Also, the label of that block will not be emitted, so this helps catch
241    // errors where we reference that label.
242    if (block->IsSingleJump()) continue;
243    Bind(block);
244    // This ensures that we have correct native line mapping for all native instructions.
245    // It is necessary to make stepping over a statement work. Otherwise, any initial
246    // instructions (e.g. moves) would be assumed to be the start of next statement.
247    MaybeRecordNativeDebugInfo(nullptr /* instruction */, block->GetDexPc());
248    for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) {
249      HInstruction* current = it.Current();
250      if (current->HasEnvironment()) {
251        // Create stackmap for HNativeDebugInfo or any instruction which calls native code.
252        // Note that we need correct mapping for the native PC of the call instruction,
253        // so the runtime's stackmap is not sufficient since it is at PC after the call.
254        MaybeRecordNativeDebugInfo(current, block->GetDexPc());
255      }
256      DisassemblyScope disassembly_scope(current, *this);
257      DCHECK(CheckTypeConsistency(current));
258      current->Accept(instruction_visitor);
259    }
260  }
261
262  GenerateSlowPaths();
263
264  // Emit catch stack maps at the end of the stack map stream as expected by the
265  // runtime exception handler.
266  if (graph_->HasTryCatch()) {
267    RecordCatchBlockInfo();
268  }
269
270  // Finalize instructions in assember;
271  Finalize(allocator);
272}
273
274void CodeGenerator::Finalize(CodeAllocator* allocator) {
275  size_t code_size = GetAssembler()->CodeSize();
276  uint8_t* buffer = allocator->Allocate(code_size);
277
278  MemoryRegion code(buffer, code_size);
279  GetAssembler()->FinalizeInstructions(code);
280}
281
282void CodeGenerator::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches ATTRIBUTE_UNUSED) {
283  // No linker patches by default.
284}
285
286void CodeGenerator::InitializeCodeGeneration(size_t number_of_spill_slots,
287                                             size_t maximum_safepoint_spill_size,
288                                             size_t number_of_out_slots,
289                                             const ArenaVector<HBasicBlock*>& block_order) {
290  block_order_ = &block_order;
291  DCHECK(!block_order.empty());
292  DCHECK(block_order[0] == GetGraph()->GetEntryBlock());
293  ComputeSpillMask();
294  first_register_slot_in_slow_path_ = RoundUp(
295      (number_of_out_slots + number_of_spill_slots) * kVRegSize, GetPreferredSlotsAlignment());
296
297  if (number_of_spill_slots == 0
298      && !HasAllocatedCalleeSaveRegisters()
299      && IsLeafMethod()
300      && !RequiresCurrentMethod()) {
301    DCHECK_EQ(maximum_safepoint_spill_size, 0u);
302    SetFrameSize(CallPushesPC() ? GetWordSize() : 0);
303  } else {
304    SetFrameSize(RoundUp(
305        first_register_slot_in_slow_path_
306        + maximum_safepoint_spill_size
307        + FrameEntrySpillSize(),
308        kStackAlignment));
309  }
310}
311
312void CodeGenerator::CreateCommonInvokeLocationSummary(
313    HInvoke* invoke, InvokeDexCallingConventionVisitor* visitor) {
314  ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena();
315  LocationSummary* locations = new (allocator) LocationSummary(invoke,
316                                                               LocationSummary::kCallOnMainOnly);
317
318  for (size_t i = 0; i < invoke->GetNumberOfArguments(); i++) {
319    HInstruction* input = invoke->InputAt(i);
320    locations->SetInAt(i, visitor->GetNextLocation(input->GetType()));
321  }
322
323  locations->SetOut(visitor->GetReturnLocation(invoke->GetType()));
324
325  if (invoke->IsInvokeStaticOrDirect()) {
326    HInvokeStaticOrDirect* call = invoke->AsInvokeStaticOrDirect();
327    switch (call->GetMethodLoadKind()) {
328      case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
329        locations->SetInAt(call->GetSpecialInputIndex(), visitor->GetMethodLocation());
330        break;
331      case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod:
332        locations->AddTemp(visitor->GetMethodLocation());
333        locations->SetInAt(call->GetSpecialInputIndex(), Location::RequiresRegister());
334        break;
335      default:
336        locations->AddTemp(visitor->GetMethodLocation());
337        break;
338    }
339  } else {
340    locations->AddTemp(visitor->GetMethodLocation());
341  }
342}
343
344void CodeGenerator::GenerateInvokeUnresolvedRuntimeCall(HInvokeUnresolved* invoke) {
345  MoveConstant(invoke->GetLocations()->GetTemp(0), invoke->GetDexMethodIndex());
346
347  // Initialize to anything to silent compiler warnings.
348  QuickEntrypointEnum entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
349  switch (invoke->GetInvokeType()) {
350    case kStatic:
351      entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
352      break;
353    case kDirect:
354      entrypoint = kQuickInvokeDirectTrampolineWithAccessCheck;
355      break;
356    case kVirtual:
357      entrypoint = kQuickInvokeVirtualTrampolineWithAccessCheck;
358      break;
359    case kSuper:
360      entrypoint = kQuickInvokeSuperTrampolineWithAccessCheck;
361      break;
362    case kInterface:
363      entrypoint = kQuickInvokeInterfaceTrampolineWithAccessCheck;
364      break;
365  }
366  InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), nullptr);
367}
368
369void CodeGenerator::CreateUnresolvedFieldLocationSummary(
370    HInstruction* field_access,
371    Primitive::Type field_type,
372    const FieldAccessCallingConvention& calling_convention) {
373  bool is_instance = field_access->IsUnresolvedInstanceFieldGet()
374      || field_access->IsUnresolvedInstanceFieldSet();
375  bool is_get = field_access->IsUnresolvedInstanceFieldGet()
376      || field_access->IsUnresolvedStaticFieldGet();
377
378  ArenaAllocator* allocator = field_access->GetBlock()->GetGraph()->GetArena();
379  LocationSummary* locations =
380      new (allocator) LocationSummary(field_access, LocationSummary::kCallOnMainOnly);
381
382  locations->AddTemp(calling_convention.GetFieldIndexLocation());
383
384  if (is_instance) {
385    // Add the `this` object for instance field accesses.
386    locations->SetInAt(0, calling_convention.GetObjectLocation());
387  }
388
389  // Note that pSetXXStatic/pGetXXStatic always takes/returns an int or int64
390  // regardless of the the type. Because of that we forced to special case
391  // the access to floating point values.
392  if (is_get) {
393    if (Primitive::IsFloatingPointType(field_type)) {
394      // The return value will be stored in regular registers while register
395      // allocator expects it in a floating point register.
396      // Note We don't need to request additional temps because the return
397      // register(s) are already blocked due the call and they may overlap with
398      // the input or field index.
399      // The transfer between the two will be done at codegen level.
400      locations->SetOut(calling_convention.GetFpuLocation(field_type));
401    } else {
402      locations->SetOut(calling_convention.GetReturnLocation(field_type));
403    }
404  } else {
405     size_t set_index = is_instance ? 1 : 0;
406     if (Primitive::IsFloatingPointType(field_type)) {
407      // The set value comes from a float location while the calling convention
408      // expects it in a regular register location. Allocate a temp for it and
409      // make the transfer at codegen.
410      AddLocationAsTemp(calling_convention.GetSetValueLocation(field_type, is_instance), locations);
411      locations->SetInAt(set_index, calling_convention.GetFpuLocation(field_type));
412    } else {
413      locations->SetInAt(set_index,
414          calling_convention.GetSetValueLocation(field_type, is_instance));
415    }
416  }
417}
418
419void CodeGenerator::GenerateUnresolvedFieldAccess(
420    HInstruction* field_access,
421    Primitive::Type field_type,
422    uint32_t field_index,
423    uint32_t dex_pc,
424    const FieldAccessCallingConvention& calling_convention) {
425  LocationSummary* locations = field_access->GetLocations();
426
427  MoveConstant(locations->GetTemp(0), field_index);
428
429  bool is_instance = field_access->IsUnresolvedInstanceFieldGet()
430      || field_access->IsUnresolvedInstanceFieldSet();
431  bool is_get = field_access->IsUnresolvedInstanceFieldGet()
432      || field_access->IsUnresolvedStaticFieldGet();
433
434  if (!is_get && Primitive::IsFloatingPointType(field_type)) {
435    // Copy the float value to be set into the calling convention register.
436    // Note that using directly the temp location is problematic as we don't
437    // support temp register pairs. To avoid boilerplate conversion code, use
438    // the location from the calling convention.
439    MoveLocation(calling_convention.GetSetValueLocation(field_type, is_instance),
440                 locations->InAt(is_instance ? 1 : 0),
441                 (Primitive::Is64BitType(field_type) ? Primitive::kPrimLong : Primitive::kPrimInt));
442  }
443
444  QuickEntrypointEnum entrypoint = kQuickSet8Static;  // Initialize to anything to avoid warnings.
445  switch (field_type) {
446    case Primitive::kPrimBoolean:
447      entrypoint = is_instance
448          ? (is_get ? kQuickGetBooleanInstance : kQuickSet8Instance)
449          : (is_get ? kQuickGetBooleanStatic : kQuickSet8Static);
450      break;
451    case Primitive::kPrimByte:
452      entrypoint = is_instance
453          ? (is_get ? kQuickGetByteInstance : kQuickSet8Instance)
454          : (is_get ? kQuickGetByteStatic : kQuickSet8Static);
455      break;
456    case Primitive::kPrimShort:
457      entrypoint = is_instance
458          ? (is_get ? kQuickGetShortInstance : kQuickSet16Instance)
459          : (is_get ? kQuickGetShortStatic : kQuickSet16Static);
460      break;
461    case Primitive::kPrimChar:
462      entrypoint = is_instance
463          ? (is_get ? kQuickGetCharInstance : kQuickSet16Instance)
464          : (is_get ? kQuickGetCharStatic : kQuickSet16Static);
465      break;
466    case Primitive::kPrimInt:
467    case Primitive::kPrimFloat:
468      entrypoint = is_instance
469          ? (is_get ? kQuickGet32Instance : kQuickSet32Instance)
470          : (is_get ? kQuickGet32Static : kQuickSet32Static);
471      break;
472    case Primitive::kPrimNot:
473      entrypoint = is_instance
474          ? (is_get ? kQuickGetObjInstance : kQuickSetObjInstance)
475          : (is_get ? kQuickGetObjStatic : kQuickSetObjStatic);
476      break;
477    case Primitive::kPrimLong:
478    case Primitive::kPrimDouble:
479      entrypoint = is_instance
480          ? (is_get ? kQuickGet64Instance : kQuickSet64Instance)
481          : (is_get ? kQuickGet64Static : kQuickSet64Static);
482      break;
483    default:
484      LOG(FATAL) << "Invalid type " << field_type;
485  }
486  InvokeRuntime(entrypoint, field_access, dex_pc, nullptr);
487
488  if (is_get && Primitive::IsFloatingPointType(field_type)) {
489    MoveLocation(locations->Out(), calling_convention.GetReturnLocation(field_type), field_type);
490  }
491}
492
493// TODO: Remove argument `code_generator_supports_read_barrier` when
494// all code generators have read barrier support.
495void CodeGenerator::CreateLoadClassLocationSummary(HLoadClass* cls,
496                                                   Location runtime_type_index_location,
497                                                   Location runtime_return_location,
498                                                   bool code_generator_supports_read_barrier) {
499  ArenaAllocator* allocator = cls->GetBlock()->GetGraph()->GetArena();
500  LocationSummary::CallKind call_kind = cls->NeedsAccessCheck()
501      ? LocationSummary::kCallOnMainOnly
502      : (((code_generator_supports_read_barrier && kEmitCompilerReadBarrier) ||
503          cls->CanCallRuntime())
504            ? LocationSummary::kCallOnSlowPath
505            : LocationSummary::kNoCall);
506  LocationSummary* locations = new (allocator) LocationSummary(cls, call_kind);
507  if (cls->NeedsAccessCheck()) {
508    locations->SetInAt(0, Location::NoLocation());
509    locations->AddTemp(runtime_type_index_location);
510    locations->SetOut(runtime_return_location);
511  } else {
512    locations->SetInAt(0, Location::RequiresRegister());
513    locations->SetOut(Location::RequiresRegister());
514  }
515}
516
517
518void CodeGenerator::BlockIfInRegister(Location location, bool is_out) const {
519  // The DCHECKS below check that a register is not specified twice in
520  // the summary. The out location can overlap with an input, so we need
521  // to special case it.
522  if (location.IsRegister()) {
523    DCHECK(is_out || !blocked_core_registers_[location.reg()]);
524    blocked_core_registers_[location.reg()] = true;
525  } else if (location.IsFpuRegister()) {
526    DCHECK(is_out || !blocked_fpu_registers_[location.reg()]);
527    blocked_fpu_registers_[location.reg()] = true;
528  } else if (location.IsFpuRegisterPair()) {
529    DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()]);
530    blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()] = true;
531    DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()]);
532    blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()] = true;
533  } else if (location.IsRegisterPair()) {
534    DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairLow<int>()]);
535    blocked_core_registers_[location.AsRegisterPairLow<int>()] = true;
536    DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairHigh<int>()]);
537    blocked_core_registers_[location.AsRegisterPairHigh<int>()] = true;
538  }
539}
540
541void CodeGenerator::AllocateLocations(HInstruction* instruction) {
542  instruction->Accept(GetLocationBuilder());
543  DCHECK(CheckTypeConsistency(instruction));
544  LocationSummary* locations = instruction->GetLocations();
545  if (!instruction->IsSuspendCheckEntry()) {
546    if (locations != nullptr) {
547      if (locations->CanCall()) {
548        MarkNotLeaf();
549      } else if (locations->Intrinsified() &&
550                 instruction->IsInvokeStaticOrDirect() &&
551                 !instruction->AsInvokeStaticOrDirect()->HasCurrentMethodInput()) {
552        // A static method call that has been fully intrinsified, and cannot call on the slow
553        // path or refer to the current method directly, no longer needs current method.
554        return;
555      }
556    }
557    if (instruction->NeedsCurrentMethod()) {
558      SetRequiresCurrentMethod();
559    }
560  }
561}
562
563void CodeGenerator::MaybeRecordStat(MethodCompilationStat compilation_stat, size_t count) const {
564  if (stats_ != nullptr) {
565    stats_->RecordStat(compilation_stat, count);
566  }
567}
568
569std::unique_ptr<CodeGenerator> CodeGenerator::Create(HGraph* graph,
570                                                     InstructionSet instruction_set,
571                                                     const InstructionSetFeatures& isa_features,
572                                                     const CompilerOptions& compiler_options,
573                                                     OptimizingCompilerStats* stats) {
574  ArenaAllocator* arena = graph->GetArena();
575  switch (instruction_set) {
576#ifdef ART_ENABLE_CODEGEN_arm
577    case kArm:
578    case kThumb2: {
579      if (kArmUseVIXL32) {
580        return std::unique_ptr<CodeGenerator>(
581            new (arena) arm::CodeGeneratorARMVIXL(graph,
582                                                  *isa_features.AsArmInstructionSetFeatures(),
583                                                  compiler_options,
584                                                  stats));
585      } else {
586          return std::unique_ptr<CodeGenerator>(
587            new (arena) arm::CodeGeneratorARM(graph,
588                                              *isa_features.AsArmInstructionSetFeatures(),
589                                              compiler_options,
590                                              stats));
591      }
592    }
593#endif
594#ifdef ART_ENABLE_CODEGEN_arm64
595    case kArm64: {
596      return std::unique_ptr<CodeGenerator>(
597          new (arena) arm64::CodeGeneratorARM64(graph,
598                                                *isa_features.AsArm64InstructionSetFeatures(),
599                                                compiler_options,
600                                                stats));
601    }
602#endif
603#ifdef ART_ENABLE_CODEGEN_mips
604    case kMips: {
605      return std::unique_ptr<CodeGenerator>(
606          new (arena) mips::CodeGeneratorMIPS(graph,
607                                              *isa_features.AsMipsInstructionSetFeatures(),
608                                              compiler_options,
609                                              stats));
610    }
611#endif
612#ifdef ART_ENABLE_CODEGEN_mips64
613    case kMips64: {
614      return std::unique_ptr<CodeGenerator>(
615          new (arena) mips64::CodeGeneratorMIPS64(graph,
616                                                  *isa_features.AsMips64InstructionSetFeatures(),
617                                                  compiler_options,
618                                                  stats));
619    }
620#endif
621#ifdef ART_ENABLE_CODEGEN_x86
622    case kX86: {
623      return std::unique_ptr<CodeGenerator>(
624          new (arena) x86::CodeGeneratorX86(graph,
625                                            *isa_features.AsX86InstructionSetFeatures(),
626                                            compiler_options,
627                                            stats));
628    }
629#endif
630#ifdef ART_ENABLE_CODEGEN_x86_64
631    case kX86_64: {
632      return std::unique_ptr<CodeGenerator>(
633          new (arena) x86_64::CodeGeneratorX86_64(graph,
634                                                  *isa_features.AsX86_64InstructionSetFeatures(),
635                                                  compiler_options,
636                                                  stats));
637    }
638#endif
639    default:
640      return nullptr;
641  }
642}
643
644size_t CodeGenerator::ComputeStackMapsSize() {
645  return stack_map_stream_.PrepareForFillIn();
646}
647
648static void CheckCovers(uint32_t dex_pc,
649                        const HGraph& graph,
650                        const CodeInfo& code_info,
651                        const ArenaVector<HSuspendCheck*>& loop_headers,
652                        ArenaVector<size_t>* covered) {
653  CodeInfoEncoding encoding = code_info.ExtractEncoding();
654  for (size_t i = 0; i < loop_headers.size(); ++i) {
655    if (loop_headers[i]->GetDexPc() == dex_pc) {
656      if (graph.IsCompilingOsr()) {
657        DCHECK(code_info.GetOsrStackMapForDexPc(dex_pc, encoding).IsValid());
658      }
659      ++(*covered)[i];
660    }
661  }
662}
663
664// Debug helper to ensure loop entries in compiled code are matched by
665// dex branch instructions.
666static void CheckLoopEntriesCanBeUsedForOsr(const HGraph& graph,
667                                            const CodeInfo& code_info,
668                                            const DexFile::CodeItem& code_item) {
669  if (graph.HasTryCatch()) {
670    // One can write loops through try/catch, which we do not support for OSR anyway.
671    return;
672  }
673  ArenaVector<HSuspendCheck*> loop_headers(graph.GetArena()->Adapter(kArenaAllocMisc));
674  for (HBasicBlock* block : graph.GetReversePostOrder()) {
675    if (block->IsLoopHeader()) {
676      HSuspendCheck* suspend_check = block->GetLoopInformation()->GetSuspendCheck();
677      if (!suspend_check->GetEnvironment()->IsFromInlinedInvoke()) {
678        loop_headers.push_back(suspend_check);
679      }
680    }
681  }
682  ArenaVector<size_t> covered(loop_headers.size(), 0, graph.GetArena()->Adapter(kArenaAllocMisc));
683  const uint16_t* code_ptr = code_item.insns_;
684  const uint16_t* code_end = code_item.insns_ + code_item.insns_size_in_code_units_;
685
686  size_t dex_pc = 0;
687  while (code_ptr < code_end) {
688    const Instruction& instruction = *Instruction::At(code_ptr);
689    if (instruction.IsBranch()) {
690      uint32_t target = dex_pc + instruction.GetTargetOffset();
691      CheckCovers(target, graph, code_info, loop_headers, &covered);
692    } else if (instruction.IsSwitch()) {
693      DexSwitchTable table(instruction, dex_pc);
694      uint16_t num_entries = table.GetNumEntries();
695      size_t offset = table.GetFirstValueIndex();
696
697      // Use a larger loop counter type to avoid overflow issues.
698      for (size_t i = 0; i < num_entries; ++i) {
699        // The target of the case.
700        uint32_t target = dex_pc + table.GetEntryAt(i + offset);
701        CheckCovers(target, graph, code_info, loop_headers, &covered);
702      }
703    }
704    dex_pc += instruction.SizeInCodeUnits();
705    code_ptr += instruction.SizeInCodeUnits();
706  }
707
708  for (size_t i = 0; i < covered.size(); ++i) {
709    DCHECK_NE(covered[i], 0u) << "Loop in compiled code has no dex branch equivalent";
710  }
711}
712
713void CodeGenerator::BuildStackMaps(MemoryRegion region, const DexFile::CodeItem& code_item) {
714  stack_map_stream_.FillIn(region);
715  if (kIsDebugBuild) {
716    CheckLoopEntriesCanBeUsedForOsr(*graph_, CodeInfo(region), code_item);
717  }
718}
719
720void CodeGenerator::RecordPcInfo(HInstruction* instruction,
721                                 uint32_t dex_pc,
722                                 SlowPathCode* slow_path) {
723  if (instruction != nullptr) {
724    // The code generated for some type conversions
725    // may call the runtime, thus normally requiring a subsequent
726    // call to this method. However, the method verifier does not
727    // produce PC information for certain instructions, which are
728    // considered "atomic" (they cannot join a GC).
729    // Therefore we do not currently record PC information for such
730    // instructions.  As this may change later, we added this special
731    // case so that code generators may nevertheless call
732    // CodeGenerator::RecordPcInfo without triggering an error in
733    // CodeGenerator::BuildNativeGCMap ("Missing ref for dex pc 0x")
734    // thereafter.
735    if (instruction->IsTypeConversion()) {
736      return;
737    }
738    if (instruction->IsRem()) {
739      Primitive::Type type = instruction->AsRem()->GetResultType();
740      if ((type == Primitive::kPrimFloat) || (type == Primitive::kPrimDouble)) {
741        return;
742      }
743    }
744  }
745
746  uint32_t outer_dex_pc = dex_pc;
747  uint32_t outer_environment_size = 0;
748  uint32_t inlining_depth = 0;
749  if (instruction != nullptr) {
750    for (HEnvironment* environment = instruction->GetEnvironment();
751         environment != nullptr;
752         environment = environment->GetParent()) {
753      outer_dex_pc = environment->GetDexPc();
754      outer_environment_size = environment->Size();
755      if (environment != instruction->GetEnvironment()) {
756        inlining_depth++;
757      }
758    }
759  }
760
761  // Collect PC infos for the mapping table.
762  uint32_t native_pc = GetAssembler()->CodePosition();
763
764  if (instruction == nullptr) {
765    // For stack overflow checks and native-debug-info entries without dex register
766    // mapping (i.e. start of basic block or start of slow path).
767    stack_map_stream_.BeginStackMapEntry(outer_dex_pc, native_pc, 0, 0, 0, 0);
768    stack_map_stream_.EndStackMapEntry();
769    return;
770  }
771  LocationSummary* locations = instruction->GetLocations();
772
773  uint32_t register_mask = locations->GetRegisterMask();
774  DCHECK_EQ(register_mask & ~locations->GetLiveRegisters()->GetCoreRegisters(), 0u);
775  if (locations->OnlyCallsOnSlowPath()) {
776    // In case of slow path, we currently set the location of caller-save registers
777    // to register (instead of their stack location when pushed before the slow-path
778    // call). Therefore register_mask contains both callee-save and caller-save
779    // registers that hold objects. We must remove the spilled caller-save from the
780    // mask, since they will be overwritten by the callee.
781    uint32_t spills = GetSlowPathSpills(locations, /* core_registers */ true);
782    register_mask &= ~spills;
783  } else {
784    // The register mask must be a subset of callee-save registers.
785    DCHECK_EQ(register_mask & core_callee_save_mask_, register_mask);
786  }
787  stack_map_stream_.BeginStackMapEntry(outer_dex_pc,
788                                       native_pc,
789                                       register_mask,
790                                       locations->GetStackMask(),
791                                       outer_environment_size,
792                                       inlining_depth);
793
794  EmitEnvironment(instruction->GetEnvironment(), slow_path);
795  stack_map_stream_.EndStackMapEntry();
796
797  HLoopInformation* info = instruction->GetBlock()->GetLoopInformation();
798  if (instruction->IsSuspendCheck() &&
799      (info != nullptr) &&
800      graph_->IsCompilingOsr() &&
801      (inlining_depth == 0)) {
802    DCHECK_EQ(info->GetSuspendCheck(), instruction);
803    // We duplicate the stack map as a marker that this stack map can be an OSR entry.
804    // Duplicating it avoids having the runtime recognize and skip an OSR stack map.
805    DCHECK(info->IsIrreducible());
806    stack_map_stream_.BeginStackMapEntry(
807        dex_pc, native_pc, register_mask, locations->GetStackMask(), outer_environment_size, 0);
808    EmitEnvironment(instruction->GetEnvironment(), slow_path);
809    stack_map_stream_.EndStackMapEntry();
810    if (kIsDebugBuild) {
811      HEnvironment* environment = instruction->GetEnvironment();
812      for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) {
813        HInstruction* in_environment = environment->GetInstructionAt(i);
814        if (in_environment != nullptr) {
815          DCHECK(in_environment->IsPhi() || in_environment->IsConstant());
816          Location location = environment->GetLocationAt(i);
817          DCHECK(location.IsStackSlot() ||
818                 location.IsDoubleStackSlot() ||
819                 location.IsConstant() ||
820                 location.IsInvalid());
821          if (location.IsStackSlot() || location.IsDoubleStackSlot()) {
822            DCHECK_LT(location.GetStackIndex(), static_cast<int32_t>(GetFrameSize()));
823          }
824        }
825      }
826    }
827  } else if (kIsDebugBuild) {
828    // Ensure stack maps are unique, by checking that the native pc in the stack map
829    // last emitted is different than the native pc of the stack map just emitted.
830    size_t number_of_stack_maps = stack_map_stream_.GetNumberOfStackMaps();
831    if (number_of_stack_maps > 1) {
832      DCHECK_NE(stack_map_stream_.GetStackMap(number_of_stack_maps - 1).native_pc_offset,
833                stack_map_stream_.GetStackMap(number_of_stack_maps - 2).native_pc_offset);
834    }
835  }
836}
837
838bool CodeGenerator::HasStackMapAtCurrentPc() {
839  uint32_t pc = GetAssembler()->CodeSize();
840  size_t count = stack_map_stream_.GetNumberOfStackMaps();
841  return count > 0 && stack_map_stream_.GetStackMap(count - 1).native_pc_offset == pc;
842}
843
844void CodeGenerator::MaybeRecordNativeDebugInfo(HInstruction* instruction,
845                                               uint32_t dex_pc,
846                                               SlowPathCode* slow_path) {
847  if (GetCompilerOptions().GetNativeDebuggable() && dex_pc != kNoDexPc) {
848    if (HasStackMapAtCurrentPc()) {
849      // Ensure that we do not collide with the stack map of the previous instruction.
850      GenerateNop();
851    }
852    RecordPcInfo(instruction, dex_pc, slow_path);
853  }
854}
855
856void CodeGenerator::RecordCatchBlockInfo() {
857  ArenaAllocator* arena = graph_->GetArena();
858
859  for (HBasicBlock* block : *block_order_) {
860    if (!block->IsCatchBlock()) {
861      continue;
862    }
863
864    uint32_t dex_pc = block->GetDexPc();
865    uint32_t num_vregs = graph_->GetNumberOfVRegs();
866    uint32_t inlining_depth = 0;  // Inlining of catch blocks is not supported at the moment.
867    uint32_t native_pc = GetAddressOf(block);
868    uint32_t register_mask = 0;   // Not used.
869
870    // The stack mask is not used, so we leave it empty.
871    ArenaBitVector* stack_mask =
872        ArenaBitVector::Create(arena, 0, /* expandable */ true, kArenaAllocCodeGenerator);
873
874    stack_map_stream_.BeginStackMapEntry(dex_pc,
875                                         native_pc,
876                                         register_mask,
877                                         stack_mask,
878                                         num_vregs,
879                                         inlining_depth);
880
881    HInstruction* current_phi = block->GetFirstPhi();
882    for (size_t vreg = 0; vreg < num_vregs; ++vreg) {
883    while (current_phi != nullptr && current_phi->AsPhi()->GetRegNumber() < vreg) {
884      HInstruction* next_phi = current_phi->GetNext();
885      DCHECK(next_phi == nullptr ||
886             current_phi->AsPhi()->GetRegNumber() <= next_phi->AsPhi()->GetRegNumber())
887          << "Phis need to be sorted by vreg number to keep this a linear-time loop.";
888      current_phi = next_phi;
889    }
890
891      if (current_phi == nullptr || current_phi->AsPhi()->GetRegNumber() != vreg) {
892        stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
893      } else {
894        Location location = current_phi->GetLiveInterval()->ToLocation();
895        switch (location.GetKind()) {
896          case Location::kStackSlot: {
897            stack_map_stream_.AddDexRegisterEntry(
898                DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
899            break;
900          }
901          case Location::kDoubleStackSlot: {
902            stack_map_stream_.AddDexRegisterEntry(
903                DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
904            stack_map_stream_.AddDexRegisterEntry(
905                DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize));
906            ++vreg;
907            DCHECK_LT(vreg, num_vregs);
908            break;
909          }
910          default: {
911            // All catch phis must be allocated to a stack slot.
912            LOG(FATAL) << "Unexpected kind " << location.GetKind();
913            UNREACHABLE();
914          }
915        }
916      }
917    }
918
919    stack_map_stream_.EndStackMapEntry();
920  }
921}
922
923void CodeGenerator::EmitEnvironment(HEnvironment* environment, SlowPathCode* slow_path) {
924  if (environment == nullptr) return;
925
926  if (environment->GetParent() != nullptr) {
927    // We emit the parent environment first.
928    EmitEnvironment(environment->GetParent(), slow_path);
929    stack_map_stream_.BeginInlineInfoEntry(environment->GetMethodIdx(),
930                                           environment->GetDexPc(),
931                                           environment->GetInvokeType(),
932                                           environment->Size());
933  }
934
935  // Walk over the environment, and record the location of dex registers.
936  for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) {
937    HInstruction* current = environment->GetInstructionAt(i);
938    if (current == nullptr) {
939      stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
940      continue;
941    }
942
943    Location location = environment->GetLocationAt(i);
944    switch (location.GetKind()) {
945      case Location::kConstant: {
946        DCHECK_EQ(current, location.GetConstant());
947        if (current->IsLongConstant()) {
948          int64_t value = current->AsLongConstant()->GetValue();
949          stack_map_stream_.AddDexRegisterEntry(
950              DexRegisterLocation::Kind::kConstant, Low32Bits(value));
951          stack_map_stream_.AddDexRegisterEntry(
952              DexRegisterLocation::Kind::kConstant, High32Bits(value));
953          ++i;
954          DCHECK_LT(i, environment_size);
955        } else if (current->IsDoubleConstant()) {
956          int64_t value = bit_cast<int64_t, double>(current->AsDoubleConstant()->GetValue());
957          stack_map_stream_.AddDexRegisterEntry(
958              DexRegisterLocation::Kind::kConstant, Low32Bits(value));
959          stack_map_stream_.AddDexRegisterEntry(
960              DexRegisterLocation::Kind::kConstant, High32Bits(value));
961          ++i;
962          DCHECK_LT(i, environment_size);
963        } else if (current->IsIntConstant()) {
964          int32_t value = current->AsIntConstant()->GetValue();
965          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, value);
966        } else if (current->IsNullConstant()) {
967          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, 0);
968        } else {
969          DCHECK(current->IsFloatConstant()) << current->DebugName();
970          int32_t value = bit_cast<int32_t, float>(current->AsFloatConstant()->GetValue());
971          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, value);
972        }
973        break;
974      }
975
976      case Location::kStackSlot: {
977        stack_map_stream_.AddDexRegisterEntry(
978            DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
979        break;
980      }
981
982      case Location::kDoubleStackSlot: {
983        stack_map_stream_.AddDexRegisterEntry(
984            DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
985        stack_map_stream_.AddDexRegisterEntry(
986            DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize));
987        ++i;
988        DCHECK_LT(i, environment_size);
989        break;
990      }
991
992      case Location::kRegister : {
993        int id = location.reg();
994        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(id)) {
995          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(id);
996          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
997          if (current->GetType() == Primitive::kPrimLong) {
998            stack_map_stream_.AddDexRegisterEntry(
999                DexRegisterLocation::Kind::kInStack, offset + kVRegSize);
1000            ++i;
1001            DCHECK_LT(i, environment_size);
1002          }
1003        } else {
1004          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, id);
1005          if (current->GetType() == Primitive::kPrimLong) {
1006            stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegisterHigh, id);
1007            ++i;
1008            DCHECK_LT(i, environment_size);
1009          }
1010        }
1011        break;
1012      }
1013
1014      case Location::kFpuRegister : {
1015        int id = location.reg();
1016        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(id)) {
1017          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(id);
1018          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1019          if (current->GetType() == Primitive::kPrimDouble) {
1020            stack_map_stream_.AddDexRegisterEntry(
1021                DexRegisterLocation::Kind::kInStack, offset + kVRegSize);
1022            ++i;
1023            DCHECK_LT(i, environment_size);
1024          }
1025        } else {
1026          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, id);
1027          if (current->GetType() == Primitive::kPrimDouble) {
1028            stack_map_stream_.AddDexRegisterEntry(
1029                DexRegisterLocation::Kind::kInFpuRegisterHigh, id);
1030            ++i;
1031            DCHECK_LT(i, environment_size);
1032          }
1033        }
1034        break;
1035      }
1036
1037      case Location::kFpuRegisterPair : {
1038        int low = location.low();
1039        int high = location.high();
1040        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(low)) {
1041          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(low);
1042          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1043        } else {
1044          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, low);
1045        }
1046        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(high)) {
1047          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(high);
1048          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1049          ++i;
1050        } else {
1051          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, high);
1052          ++i;
1053        }
1054        DCHECK_LT(i, environment_size);
1055        break;
1056      }
1057
1058      case Location::kRegisterPair : {
1059        int low = location.low();
1060        int high = location.high();
1061        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(low)) {
1062          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(low);
1063          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1064        } else {
1065          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, low);
1066        }
1067        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(high)) {
1068          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(high);
1069          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1070        } else {
1071          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, high);
1072        }
1073        ++i;
1074        DCHECK_LT(i, environment_size);
1075        break;
1076      }
1077
1078      case Location::kInvalid: {
1079        stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
1080        break;
1081      }
1082
1083      default:
1084        LOG(FATAL) << "Unexpected kind " << location.GetKind();
1085    }
1086  }
1087
1088  if (environment->GetParent() != nullptr) {
1089    stack_map_stream_.EndInlineInfoEntry();
1090  }
1091}
1092
1093bool CodeGenerator::CanMoveNullCheckToUser(HNullCheck* null_check) {
1094  HInstruction* first_next_not_move = null_check->GetNextDisregardingMoves();
1095
1096  return (first_next_not_move != nullptr)
1097      && first_next_not_move->CanDoImplicitNullCheckOn(null_check->InputAt(0));
1098}
1099
1100void CodeGenerator::MaybeRecordImplicitNullCheck(HInstruction* instr) {
1101  if (!compiler_options_.GetImplicitNullChecks()) {
1102    return;
1103  }
1104
1105  // If we are from a static path don't record the pc as we can't throw NPE.
1106  // NB: having the checks here makes the code much less verbose in the arch
1107  // specific code generators.
1108  if (instr->IsStaticFieldSet() || instr->IsStaticFieldGet()) {
1109    return;
1110  }
1111
1112  if (!instr->CanDoImplicitNullCheckOn(instr->InputAt(0))) {
1113    return;
1114  }
1115
1116  // Find the first previous instruction which is not a move.
1117  HInstruction* first_prev_not_move = instr->GetPreviousDisregardingMoves();
1118
1119  // If the instruction is a null check it means that `instr` is the first user
1120  // and needs to record the pc.
1121  if (first_prev_not_move != nullptr && first_prev_not_move->IsNullCheck()) {
1122    HNullCheck* null_check = first_prev_not_move->AsNullCheck();
1123    // TODO: The parallel moves modify the environment. Their changes need to be
1124    // reverted otherwise the stack maps at the throw point will not be correct.
1125    RecordPcInfo(null_check, null_check->GetDexPc());
1126  }
1127}
1128
1129LocationSummary* CodeGenerator::CreateThrowingSlowPathLocations(HInstruction* instruction,
1130                                                                RegisterSet caller_saves) {
1131  // Note: Using kNoCall allows the method to be treated as leaf (and eliminate the
1132  // HSuspendCheck from entry block). However, it will still get a valid stack frame
1133  // because the HNullCheck needs an environment.
1134  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
1135  // When throwing from a try block, we may need to retrieve dalvik registers from
1136  // physical registers and we also need to set up stack mask for GC. This is
1137  // implicitly achieved by passing kCallOnSlowPath to the LocationSummary.
1138  bool can_throw_into_catch_block = instruction->CanThrowIntoCatchBlock();
1139  if (can_throw_into_catch_block) {
1140    call_kind = LocationSummary::kCallOnSlowPath;
1141  }
1142  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
1143  if (can_throw_into_catch_block && compiler_options_.GetImplicitNullChecks()) {
1144    locations->SetCustomSlowPathCallerSaves(caller_saves);  // Default: no caller-save registers.
1145  }
1146  DCHECK(!instruction->HasUses());
1147  return locations;
1148}
1149
1150void CodeGenerator::GenerateNullCheck(HNullCheck* instruction) {
1151  if (compiler_options_.GetImplicitNullChecks()) {
1152    MaybeRecordStat(kImplicitNullCheckGenerated);
1153    GenerateImplicitNullCheck(instruction);
1154  } else {
1155    MaybeRecordStat(kExplicitNullCheckGenerated);
1156    GenerateExplicitNullCheck(instruction);
1157  }
1158}
1159
1160void CodeGenerator::ClearSpillSlotsFromLoopPhisInStackMap(HSuspendCheck* suspend_check) const {
1161  LocationSummary* locations = suspend_check->GetLocations();
1162  HBasicBlock* block = suspend_check->GetBlock();
1163  DCHECK(block->GetLoopInformation()->GetSuspendCheck() == suspend_check);
1164  DCHECK(block->IsLoopHeader());
1165
1166  for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) {
1167    HInstruction* current = it.Current();
1168    LiveInterval* interval = current->GetLiveInterval();
1169    // We only need to clear bits of loop phis containing objects and allocated in register.
1170    // Loop phis allocated on stack already have the object in the stack.
1171    if (current->GetType() == Primitive::kPrimNot
1172        && interval->HasRegister()
1173        && interval->HasSpillSlot()) {
1174      locations->ClearStackBit(interval->GetSpillSlot() / kVRegSize);
1175    }
1176  }
1177}
1178
1179void CodeGenerator::EmitParallelMoves(Location from1,
1180                                      Location to1,
1181                                      Primitive::Type type1,
1182                                      Location from2,
1183                                      Location to2,
1184                                      Primitive::Type type2) {
1185  HParallelMove parallel_move(GetGraph()->GetArena());
1186  parallel_move.AddMove(from1, to1, type1, nullptr);
1187  parallel_move.AddMove(from2, to2, type2, nullptr);
1188  GetMoveResolver()->EmitNativeCode(&parallel_move);
1189}
1190
1191void CodeGenerator::ValidateInvokeRuntime(QuickEntrypointEnum entrypoint,
1192                                          HInstruction* instruction,
1193                                          SlowPathCode* slow_path) {
1194  // Ensure that the call kind indication given to the register allocator is
1195  // coherent with the runtime call generated.
1196  if (slow_path == nullptr) {
1197    DCHECK(instruction->GetLocations()->WillCall())
1198        << "instruction->DebugName()=" << instruction->DebugName();
1199  } else {
1200    DCHECK(instruction->GetLocations()->CallsOnSlowPath() || slow_path->IsFatal())
1201        << "instruction->DebugName()=" << instruction->DebugName()
1202        << " slow_path->GetDescription()=" << slow_path->GetDescription();
1203  }
1204
1205  // Check that the GC side effect is set when required.
1206  // TODO: Reverse EntrypointCanTriggerGC
1207  if (EntrypointCanTriggerGC(entrypoint)) {
1208    if (slow_path == nullptr) {
1209      DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()))
1210          << "instruction->DebugName()=" << instruction->DebugName()
1211          << " instruction->GetSideEffects().ToString()="
1212          << instruction->GetSideEffects().ToString();
1213    } else {
1214      DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()) ||
1215             // When (non-Baker) read barriers are enabled, some instructions
1216             // use a slow path to emit a read barrier, which does not trigger
1217             // GC.
1218             (kEmitCompilerReadBarrier &&
1219              !kUseBakerReadBarrier &&
1220              (instruction->IsInstanceFieldGet() ||
1221               instruction->IsStaticFieldGet() ||
1222               instruction->IsArrayGet() ||
1223               instruction->IsLoadClass() ||
1224               instruction->IsLoadString() ||
1225               instruction->IsInstanceOf() ||
1226               instruction->IsCheckCast() ||
1227               (instruction->IsInvokeVirtual() && instruction->GetLocations()->Intrinsified()))))
1228          << "instruction->DebugName()=" << instruction->DebugName()
1229          << " instruction->GetSideEffects().ToString()="
1230          << instruction->GetSideEffects().ToString()
1231          << " slow_path->GetDescription()=" << slow_path->GetDescription();
1232    }
1233  } else {
1234    // The GC side effect is not required for the instruction. But the instruction might still have
1235    // it, for example if it calls other entrypoints requiring it.
1236  }
1237
1238  // Check the coherency of leaf information.
1239  DCHECK(instruction->IsSuspendCheck()
1240         || ((slow_path != nullptr) && slow_path->IsFatal())
1241         || instruction->GetLocations()->CanCall()
1242         || !IsLeafMethod())
1243      << instruction->DebugName() << ((slow_path != nullptr) ? slow_path->GetDescription() : "");
1244}
1245
1246void CodeGenerator::ValidateInvokeRuntimeWithoutRecordingPcInfo(HInstruction* instruction,
1247                                                                SlowPathCode* slow_path) {
1248  DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath())
1249      << "instruction->DebugName()=" << instruction->DebugName()
1250      << " slow_path->GetDescription()=" << slow_path->GetDescription();
1251  // Only the Baker read barrier marking slow path used by certains
1252  // instructions is expected to invoke the runtime without recording
1253  // PC-related information.
1254  DCHECK(kUseBakerReadBarrier);
1255  DCHECK(instruction->IsInstanceFieldGet() ||
1256         instruction->IsStaticFieldGet() ||
1257         instruction->IsArrayGet() ||
1258         instruction->IsArraySet() ||
1259         instruction->IsLoadClass() ||
1260         instruction->IsLoadString() ||
1261         instruction->IsInstanceOf() ||
1262         instruction->IsCheckCast() ||
1263         (instruction->IsInvokeVirtual() && instruction->GetLocations()->Intrinsified()) ||
1264         (instruction->IsInvokeStaticOrDirect() && instruction->GetLocations()->Intrinsified()))
1265      << "instruction->DebugName()=" << instruction->DebugName()
1266      << " slow_path->GetDescription()=" << slow_path->GetDescription();
1267}
1268
1269void SlowPathCode::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
1270  size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
1271
1272  const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true);
1273  for (uint32_t i : LowToHighBits(core_spills)) {
1274    // If the register holds an object, update the stack mask.
1275    if (locations->RegisterContainsObject(i)) {
1276      locations->SetStackBit(stack_offset / kVRegSize);
1277    }
1278    DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1279    DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1280    saved_core_stack_offsets_[i] = stack_offset;
1281    stack_offset += codegen->SaveCoreRegister(stack_offset, i);
1282  }
1283
1284  const uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false);
1285  for (uint32_t i : LowToHighBits(fp_spills)) {
1286    DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1287    DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1288    saved_fpu_stack_offsets_[i] = stack_offset;
1289    stack_offset += codegen->SaveFloatingPointRegister(stack_offset, i);
1290  }
1291}
1292
1293void SlowPathCode::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
1294  size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
1295
1296  const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true);
1297  for (uint32_t i : LowToHighBits(core_spills)) {
1298    DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1299    DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1300    stack_offset += codegen->RestoreCoreRegister(stack_offset, i);
1301  }
1302
1303  const uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false);
1304  for (uint32_t i : LowToHighBits(fp_spills)) {
1305    DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1306    DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1307    stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, i);
1308  }
1309}
1310
1311void CodeGenerator::CreateSystemArrayCopyLocationSummary(HInvoke* invoke) {
1312  // Check to see if we have known failures that will cause us to have to bail out
1313  // to the runtime, and just generate the runtime call directly.
1314  HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant();
1315  HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant();
1316
1317  // The positions must be non-negative.
1318  if ((src_pos != nullptr && src_pos->GetValue() < 0) ||
1319      (dest_pos != nullptr && dest_pos->GetValue() < 0)) {
1320    // We will have to fail anyways.
1321    return;
1322  }
1323
1324  // The length must be >= 0.
1325  HIntConstant* length = invoke->InputAt(4)->AsIntConstant();
1326  if (length != nullptr) {
1327    int32_t len = length->GetValue();
1328    if (len < 0) {
1329      // Just call as normal.
1330      return;
1331    }
1332  }
1333
1334  SystemArrayCopyOptimizations optimizations(invoke);
1335
1336  if (optimizations.GetDestinationIsSource()) {
1337    if (src_pos != nullptr && dest_pos != nullptr && src_pos->GetValue() < dest_pos->GetValue()) {
1338      // We only support backward copying if source and destination are the same.
1339      return;
1340    }
1341  }
1342
1343  if (optimizations.GetDestinationIsPrimitiveArray() || optimizations.GetSourceIsPrimitiveArray()) {
1344    // We currently don't intrinsify primitive copying.
1345    return;
1346  }
1347
1348  ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena();
1349  LocationSummary* locations = new (allocator) LocationSummary(invoke,
1350                                                               LocationSummary::kCallOnSlowPath,
1351                                                               kIntrinsified);
1352  // arraycopy(Object src, int src_pos, Object dest, int dest_pos, int length).
1353  locations->SetInAt(0, Location::RequiresRegister());
1354  locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1)));
1355  locations->SetInAt(2, Location::RequiresRegister());
1356  locations->SetInAt(3, Location::RegisterOrConstant(invoke->InputAt(3)));
1357  locations->SetInAt(4, Location::RegisterOrConstant(invoke->InputAt(4)));
1358
1359  locations->AddTemp(Location::RequiresRegister());
1360  locations->AddTemp(Location::RequiresRegister());
1361  locations->AddTemp(Location::RequiresRegister());
1362}
1363
1364uint32_t CodeGenerator::GetReferenceSlowFlagOffset() const {
1365  ScopedObjectAccess soa(Thread::Current());
1366  mirror::Class* klass = mirror::Reference::GetJavaLangRefReference();
1367  DCHECK(klass->IsInitialized());
1368  return klass->GetSlowPathFlagOffset().Uint32Value();
1369}
1370
1371uint32_t CodeGenerator::GetReferenceDisableFlagOffset() const {
1372  ScopedObjectAccess soa(Thread::Current());
1373  mirror::Class* klass = mirror::Reference::GetJavaLangRefReference();
1374  DCHECK(klass->IsInitialized());
1375  return klass->GetDisableIntrinsicFlagOffset().Uint32Value();
1376}
1377
1378}  // namespace art
1379