code_generator.cc revision fa4333dcb481e564f54726b4e6f8153612df835e 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator.h"
18
19#ifdef ART_ENABLE_CODEGEN_arm
20#include "code_generator_arm.h"
21#include "code_generator_arm_vixl.h"
22#endif
23
24#ifdef ART_ENABLE_CODEGEN_arm64
25#include "code_generator_arm64.h"
26#endif
27
28#ifdef ART_ENABLE_CODEGEN_x86
29#include "code_generator_x86.h"
30#endif
31
32#ifdef ART_ENABLE_CODEGEN_x86_64
33#include "code_generator_x86_64.h"
34#endif
35
36#ifdef ART_ENABLE_CODEGEN_mips
37#include "code_generator_mips.h"
38#endif
39
40#ifdef ART_ENABLE_CODEGEN_mips64
41#include "code_generator_mips64.h"
42#endif
43
44#include "bytecode_utils.h"
45#include "compiled_method.h"
46#include "dex/verified_method.h"
47#include "driver/compiler_driver.h"
48#include "graph_visualizer.h"
49#include "intrinsics.h"
50#include "leb128.h"
51#include "mirror/array-inl.h"
52#include "mirror/object_array-inl.h"
53#include "mirror/object_reference.h"
54#include "mirror/string.h"
55#include "parallel_move_resolver.h"
56#include "ssa_liveness_analysis.h"
57#include "utils/assembler.h"
58
59namespace art {
60
61// Return whether a location is consistent with a type.
62static bool CheckType(Primitive::Type type, Location location) {
63  if (location.IsFpuRegister()
64      || (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresFpuRegister))) {
65    return (type == Primitive::kPrimFloat) || (type == Primitive::kPrimDouble);
66  } else if (location.IsRegister() ||
67             (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresRegister))) {
68    return Primitive::IsIntegralType(type) || (type == Primitive::kPrimNot);
69  } else if (location.IsRegisterPair()) {
70    return type == Primitive::kPrimLong;
71  } else if (location.IsFpuRegisterPair()) {
72    return type == Primitive::kPrimDouble;
73  } else if (location.IsStackSlot()) {
74    return (Primitive::IsIntegralType(type) && type != Primitive::kPrimLong)
75           || (type == Primitive::kPrimFloat)
76           || (type == Primitive::kPrimNot);
77  } else if (location.IsDoubleStackSlot()) {
78    return (type == Primitive::kPrimLong) || (type == Primitive::kPrimDouble);
79  } else if (location.IsConstant()) {
80    if (location.GetConstant()->IsIntConstant()) {
81      return Primitive::IsIntegralType(type) && (type != Primitive::kPrimLong);
82    } else if (location.GetConstant()->IsNullConstant()) {
83      return type == Primitive::kPrimNot;
84    } else if (location.GetConstant()->IsLongConstant()) {
85      return type == Primitive::kPrimLong;
86    } else if (location.GetConstant()->IsFloatConstant()) {
87      return type == Primitive::kPrimFloat;
88    } else {
89      return location.GetConstant()->IsDoubleConstant()
90          && (type == Primitive::kPrimDouble);
91    }
92  } else {
93    return location.IsInvalid() || (location.GetPolicy() == Location::kAny);
94  }
95}
96
97// Check that a location summary is consistent with an instruction.
98static bool CheckTypeConsistency(HInstruction* instruction) {
99  LocationSummary* locations = instruction->GetLocations();
100  if (locations == nullptr) {
101    return true;
102  }
103
104  if (locations->Out().IsUnallocated()
105      && (locations->Out().GetPolicy() == Location::kSameAsFirstInput)) {
106    DCHECK(CheckType(instruction->GetType(), locations->InAt(0)))
107        << instruction->GetType()
108        << " " << locations->InAt(0);
109  } else {
110    DCHECK(CheckType(instruction->GetType(), locations->Out()))
111        << instruction->GetType()
112        << " " << locations->Out();
113  }
114
115  HConstInputsRef inputs = instruction->GetInputs();
116  for (size_t i = 0; i < inputs.size(); ++i) {
117    DCHECK(CheckType(inputs[i]->GetType(), locations->InAt(i)))
118      << inputs[i]->GetType() << " " << locations->InAt(i);
119  }
120
121  HEnvironment* environment = instruction->GetEnvironment();
122  for (size_t i = 0; i < instruction->EnvironmentSize(); ++i) {
123    if (environment->GetInstructionAt(i) != nullptr) {
124      Primitive::Type type = environment->GetInstructionAt(i)->GetType();
125      DCHECK(CheckType(type, environment->GetLocationAt(i)))
126        << type << " " << environment->GetLocationAt(i);
127    } else {
128      DCHECK(environment->GetLocationAt(i).IsInvalid())
129        << environment->GetLocationAt(i);
130    }
131  }
132  return true;
133}
134
135size_t CodeGenerator::GetCacheOffset(uint32_t index) {
136  return sizeof(GcRoot<mirror::Object>) * index;
137}
138
139size_t CodeGenerator::GetCachePointerOffset(uint32_t index) {
140  auto pointer_size = InstructionSetPointerSize(GetInstructionSet());
141  return static_cast<size_t>(pointer_size) * index;
142}
143
144uint32_t CodeGenerator::GetArrayLengthOffset(HArrayLength* array_length) {
145  return array_length->IsStringLength()
146      ? mirror::String::CountOffset().Uint32Value()
147      : mirror::Array::LengthOffset().Uint32Value();
148}
149
150uint32_t CodeGenerator::GetArrayDataOffset(HArrayGet* array_get) {
151  DCHECK(array_get->GetType() == Primitive::kPrimChar || !array_get->IsStringCharAt());
152  return array_get->IsStringCharAt()
153      ? mirror::String::ValueOffset().Uint32Value()
154      : mirror::Array::DataOffset(Primitive::ComponentSize(array_get->GetType())).Uint32Value();
155}
156
157bool CodeGenerator::GoesToNextBlock(HBasicBlock* current, HBasicBlock* next) const {
158  DCHECK_EQ((*block_order_)[current_block_index_], current);
159  return GetNextBlockToEmit() == FirstNonEmptyBlock(next);
160}
161
162HBasicBlock* CodeGenerator::GetNextBlockToEmit() const {
163  for (size_t i = current_block_index_ + 1; i < block_order_->size(); ++i) {
164    HBasicBlock* block = (*block_order_)[i];
165    if (!block->IsSingleJump()) {
166      return block;
167    }
168  }
169  return nullptr;
170}
171
172HBasicBlock* CodeGenerator::FirstNonEmptyBlock(HBasicBlock* block) const {
173  while (block->IsSingleJump()) {
174    block = block->GetSuccessors()[0];
175  }
176  return block;
177}
178
179class DisassemblyScope {
180 public:
181  DisassemblyScope(HInstruction* instruction, const CodeGenerator& codegen)
182      : codegen_(codegen), instruction_(instruction), start_offset_(static_cast<size_t>(-1)) {
183    if (codegen_.GetDisassemblyInformation() != nullptr) {
184      start_offset_ = codegen_.GetAssembler().CodeSize();
185    }
186  }
187
188  ~DisassemblyScope() {
189    // We avoid building this data when we know it will not be used.
190    if (codegen_.GetDisassemblyInformation() != nullptr) {
191      codegen_.GetDisassemblyInformation()->AddInstructionInterval(
192          instruction_, start_offset_, codegen_.GetAssembler().CodeSize());
193    }
194  }
195
196 private:
197  const CodeGenerator& codegen_;
198  HInstruction* instruction_;
199  size_t start_offset_;
200};
201
202
203void CodeGenerator::GenerateSlowPaths() {
204  size_t code_start = 0;
205  for (const std::unique_ptr<SlowPathCode>& slow_path_unique_ptr : slow_paths_) {
206    SlowPathCode* slow_path = slow_path_unique_ptr.get();
207    current_slow_path_ = slow_path;
208    if (disasm_info_ != nullptr) {
209      code_start = GetAssembler()->CodeSize();
210    }
211    // Record the dex pc at start of slow path (required for java line number mapping).
212    MaybeRecordNativeDebugInfo(slow_path->GetInstruction(), slow_path->GetDexPc(), slow_path);
213    slow_path->EmitNativeCode(this);
214    if (disasm_info_ != nullptr) {
215      disasm_info_->AddSlowPathInterval(slow_path, code_start, GetAssembler()->CodeSize());
216    }
217  }
218  current_slow_path_ = nullptr;
219}
220
221void CodeGenerator::Compile(CodeAllocator* allocator) {
222  // The register allocator already called `InitializeCodeGeneration`,
223  // where the frame size has been computed.
224  DCHECK(block_order_ != nullptr);
225  Initialize();
226
227  HGraphVisitor* instruction_visitor = GetInstructionVisitor();
228  DCHECK_EQ(current_block_index_, 0u);
229
230  size_t frame_start = GetAssembler()->CodeSize();
231  GenerateFrameEntry();
232  DCHECK_EQ(GetAssembler()->cfi().GetCurrentCFAOffset(), static_cast<int>(frame_size_));
233  if (disasm_info_ != nullptr) {
234    disasm_info_->SetFrameEntryInterval(frame_start, GetAssembler()->CodeSize());
235  }
236
237  for (size_t e = block_order_->size(); current_block_index_ < e; ++current_block_index_) {
238    HBasicBlock* block = (*block_order_)[current_block_index_];
239    // Don't generate code for an empty block. Its predecessors will branch to its successor
240    // directly. Also, the label of that block will not be emitted, so this helps catch
241    // errors where we reference that label.
242    if (block->IsSingleJump()) continue;
243    Bind(block);
244    // This ensures that we have correct native line mapping for all native instructions.
245    // It is necessary to make stepping over a statement work. Otherwise, any initial
246    // instructions (e.g. moves) would be assumed to be the start of next statement.
247    MaybeRecordNativeDebugInfo(nullptr /* instruction */, block->GetDexPc());
248    for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) {
249      HInstruction* current = it.Current();
250      if (current->HasEnvironment()) {
251        // Create stackmap for HNativeDebugInfo or any instruction which calls native code.
252        // Note that we need correct mapping for the native PC of the call instruction,
253        // so the runtime's stackmap is not sufficient since it is at PC after the call.
254        MaybeRecordNativeDebugInfo(current, block->GetDexPc());
255      }
256      DisassemblyScope disassembly_scope(current, *this);
257      DCHECK(CheckTypeConsistency(current));
258      current->Accept(instruction_visitor);
259    }
260  }
261
262  GenerateSlowPaths();
263
264  // Emit catch stack maps at the end of the stack map stream as expected by the
265  // runtime exception handler.
266  if (graph_->HasTryCatch()) {
267    RecordCatchBlockInfo();
268  }
269
270  // Finalize instructions in assember;
271  Finalize(allocator);
272}
273
274void CodeGenerator::Finalize(CodeAllocator* allocator) {
275  size_t code_size = GetAssembler()->CodeSize();
276  uint8_t* buffer = allocator->Allocate(code_size);
277
278  MemoryRegion code(buffer, code_size);
279  GetAssembler()->FinalizeInstructions(code);
280}
281
282void CodeGenerator::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches ATTRIBUTE_UNUSED) {
283  // No linker patches by default.
284}
285
286void CodeGenerator::InitializeCodeGeneration(size_t number_of_spill_slots,
287                                             size_t maximum_safepoint_spill_size,
288                                             size_t number_of_out_slots,
289                                             const ArenaVector<HBasicBlock*>& block_order) {
290  block_order_ = &block_order;
291  DCHECK(!block_order.empty());
292  DCHECK(block_order[0] == GetGraph()->GetEntryBlock());
293  ComputeSpillMask();
294  first_register_slot_in_slow_path_ = RoundUp(
295      (number_of_out_slots + number_of_spill_slots) * kVRegSize, GetPreferredSlotsAlignment());
296
297  if (number_of_spill_slots == 0
298      && !HasAllocatedCalleeSaveRegisters()
299      && IsLeafMethod()
300      && !RequiresCurrentMethod()) {
301    DCHECK_EQ(maximum_safepoint_spill_size, 0u);
302    SetFrameSize(CallPushesPC() ? GetWordSize() : 0);
303  } else {
304    SetFrameSize(RoundUp(
305        first_register_slot_in_slow_path_
306        + maximum_safepoint_spill_size
307        + (GetGraph()->HasShouldDeoptimizeFlag() ? kShouldDeoptimizeFlagSize : 0)
308        + FrameEntrySpillSize(),
309        kStackAlignment));
310  }
311}
312
313void CodeGenerator::CreateCommonInvokeLocationSummary(
314    HInvoke* invoke, InvokeDexCallingConventionVisitor* visitor) {
315  ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena();
316  LocationSummary* locations = new (allocator) LocationSummary(invoke,
317                                                               LocationSummary::kCallOnMainOnly);
318
319  for (size_t i = 0; i < invoke->GetNumberOfArguments(); i++) {
320    HInstruction* input = invoke->InputAt(i);
321    locations->SetInAt(i, visitor->GetNextLocation(input->GetType()));
322  }
323
324  locations->SetOut(visitor->GetReturnLocation(invoke->GetType()));
325
326  if (invoke->IsInvokeStaticOrDirect()) {
327    HInvokeStaticOrDirect* call = invoke->AsInvokeStaticOrDirect();
328    switch (call->GetMethodLoadKind()) {
329      case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
330        locations->SetInAt(call->GetSpecialInputIndex(), visitor->GetMethodLocation());
331        break;
332      case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod:
333        locations->AddTemp(visitor->GetMethodLocation());
334        locations->SetInAt(call->GetSpecialInputIndex(), Location::RequiresRegister());
335        break;
336      default:
337        locations->AddTemp(visitor->GetMethodLocation());
338        break;
339    }
340  } else {
341    locations->AddTemp(visitor->GetMethodLocation());
342  }
343}
344
345void CodeGenerator::GenerateInvokeUnresolvedRuntimeCall(HInvokeUnresolved* invoke) {
346  MoveConstant(invoke->GetLocations()->GetTemp(0), invoke->GetDexMethodIndex());
347
348  // Initialize to anything to silent compiler warnings.
349  QuickEntrypointEnum entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
350  switch (invoke->GetInvokeType()) {
351    case kStatic:
352      entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
353      break;
354    case kDirect:
355      entrypoint = kQuickInvokeDirectTrampolineWithAccessCheck;
356      break;
357    case kVirtual:
358      entrypoint = kQuickInvokeVirtualTrampolineWithAccessCheck;
359      break;
360    case kSuper:
361      entrypoint = kQuickInvokeSuperTrampolineWithAccessCheck;
362      break;
363    case kInterface:
364      entrypoint = kQuickInvokeInterfaceTrampolineWithAccessCheck;
365      break;
366  }
367  InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), nullptr);
368}
369
370void CodeGenerator::GenerateInvokePolymorphicCall(HInvokePolymorphic* invoke) {
371  MoveConstant(invoke->GetLocations()->GetTemp(0), static_cast<int32_t>(invoke->GetType()));
372  QuickEntrypointEnum entrypoint = kQuickInvokePolymorphic;
373  InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), nullptr);
374}
375
376void CodeGenerator::CreateUnresolvedFieldLocationSummary(
377    HInstruction* field_access,
378    Primitive::Type field_type,
379    const FieldAccessCallingConvention& calling_convention) {
380  bool is_instance = field_access->IsUnresolvedInstanceFieldGet()
381      || field_access->IsUnresolvedInstanceFieldSet();
382  bool is_get = field_access->IsUnresolvedInstanceFieldGet()
383      || field_access->IsUnresolvedStaticFieldGet();
384
385  ArenaAllocator* allocator = field_access->GetBlock()->GetGraph()->GetArena();
386  LocationSummary* locations =
387      new (allocator) LocationSummary(field_access, LocationSummary::kCallOnMainOnly);
388
389  locations->AddTemp(calling_convention.GetFieldIndexLocation());
390
391  if (is_instance) {
392    // Add the `this` object for instance field accesses.
393    locations->SetInAt(0, calling_convention.GetObjectLocation());
394  }
395
396  // Note that pSetXXStatic/pGetXXStatic always takes/returns an int or int64
397  // regardless of the the type. Because of that we forced to special case
398  // the access to floating point values.
399  if (is_get) {
400    if (Primitive::IsFloatingPointType(field_type)) {
401      // The return value will be stored in regular registers while register
402      // allocator expects it in a floating point register.
403      // Note We don't need to request additional temps because the return
404      // register(s) are already blocked due the call and they may overlap with
405      // the input or field index.
406      // The transfer between the two will be done at codegen level.
407      locations->SetOut(calling_convention.GetFpuLocation(field_type));
408    } else {
409      locations->SetOut(calling_convention.GetReturnLocation(field_type));
410    }
411  } else {
412     size_t set_index = is_instance ? 1 : 0;
413     if (Primitive::IsFloatingPointType(field_type)) {
414      // The set value comes from a float location while the calling convention
415      // expects it in a regular register location. Allocate a temp for it and
416      // make the transfer at codegen.
417      AddLocationAsTemp(calling_convention.GetSetValueLocation(field_type, is_instance), locations);
418      locations->SetInAt(set_index, calling_convention.GetFpuLocation(field_type));
419    } else {
420      locations->SetInAt(set_index,
421          calling_convention.GetSetValueLocation(field_type, is_instance));
422    }
423  }
424}
425
426void CodeGenerator::GenerateUnresolvedFieldAccess(
427    HInstruction* field_access,
428    Primitive::Type field_type,
429    uint32_t field_index,
430    uint32_t dex_pc,
431    const FieldAccessCallingConvention& calling_convention) {
432  LocationSummary* locations = field_access->GetLocations();
433
434  MoveConstant(locations->GetTemp(0), field_index);
435
436  bool is_instance = field_access->IsUnresolvedInstanceFieldGet()
437      || field_access->IsUnresolvedInstanceFieldSet();
438  bool is_get = field_access->IsUnresolvedInstanceFieldGet()
439      || field_access->IsUnresolvedStaticFieldGet();
440
441  if (!is_get && Primitive::IsFloatingPointType(field_type)) {
442    // Copy the float value to be set into the calling convention register.
443    // Note that using directly the temp location is problematic as we don't
444    // support temp register pairs. To avoid boilerplate conversion code, use
445    // the location from the calling convention.
446    MoveLocation(calling_convention.GetSetValueLocation(field_type, is_instance),
447                 locations->InAt(is_instance ? 1 : 0),
448                 (Primitive::Is64BitType(field_type) ? Primitive::kPrimLong : Primitive::kPrimInt));
449  }
450
451  QuickEntrypointEnum entrypoint = kQuickSet8Static;  // Initialize to anything to avoid warnings.
452  switch (field_type) {
453    case Primitive::kPrimBoolean:
454      entrypoint = is_instance
455          ? (is_get ? kQuickGetBooleanInstance : kQuickSet8Instance)
456          : (is_get ? kQuickGetBooleanStatic : kQuickSet8Static);
457      break;
458    case Primitive::kPrimByte:
459      entrypoint = is_instance
460          ? (is_get ? kQuickGetByteInstance : kQuickSet8Instance)
461          : (is_get ? kQuickGetByteStatic : kQuickSet8Static);
462      break;
463    case Primitive::kPrimShort:
464      entrypoint = is_instance
465          ? (is_get ? kQuickGetShortInstance : kQuickSet16Instance)
466          : (is_get ? kQuickGetShortStatic : kQuickSet16Static);
467      break;
468    case Primitive::kPrimChar:
469      entrypoint = is_instance
470          ? (is_get ? kQuickGetCharInstance : kQuickSet16Instance)
471          : (is_get ? kQuickGetCharStatic : kQuickSet16Static);
472      break;
473    case Primitive::kPrimInt:
474    case Primitive::kPrimFloat:
475      entrypoint = is_instance
476          ? (is_get ? kQuickGet32Instance : kQuickSet32Instance)
477          : (is_get ? kQuickGet32Static : kQuickSet32Static);
478      break;
479    case Primitive::kPrimNot:
480      entrypoint = is_instance
481          ? (is_get ? kQuickGetObjInstance : kQuickSetObjInstance)
482          : (is_get ? kQuickGetObjStatic : kQuickSetObjStatic);
483      break;
484    case Primitive::kPrimLong:
485    case Primitive::kPrimDouble:
486      entrypoint = is_instance
487          ? (is_get ? kQuickGet64Instance : kQuickSet64Instance)
488          : (is_get ? kQuickGet64Static : kQuickSet64Static);
489      break;
490    default:
491      LOG(FATAL) << "Invalid type " << field_type;
492  }
493  InvokeRuntime(entrypoint, field_access, dex_pc, nullptr);
494
495  if (is_get && Primitive::IsFloatingPointType(field_type)) {
496    MoveLocation(locations->Out(), calling_convention.GetReturnLocation(field_type), field_type);
497  }
498}
499
500void CodeGenerator::CreateLoadClassRuntimeCallLocationSummary(HLoadClass* cls,
501                                                              Location runtime_type_index_location,
502                                                              Location runtime_return_location) {
503  DCHECK_EQ(cls->GetLoadKind(), HLoadClass::LoadKind::kDexCacheViaMethod);
504  DCHECK_EQ(cls->InputCount(), 1u);
505  LocationSummary* locations = new (cls->GetBlock()->GetGraph()->GetArena()) LocationSummary(
506      cls, LocationSummary::kCallOnMainOnly);
507  locations->SetInAt(0, Location::NoLocation());
508  locations->AddTemp(runtime_type_index_location);
509  locations->SetOut(runtime_return_location);
510}
511
512void CodeGenerator::GenerateLoadClassRuntimeCall(HLoadClass* cls) {
513  DCHECK_EQ(cls->GetLoadKind(), HLoadClass::LoadKind::kDexCacheViaMethod);
514  LocationSummary* locations = cls->GetLocations();
515  MoveConstant(locations->GetTemp(0), cls->GetTypeIndex().index_);
516  if (cls->NeedsAccessCheck()) {
517    CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>();
518    InvokeRuntime(kQuickInitializeTypeAndVerifyAccess, cls, cls->GetDexPc());
519  } else if (cls->MustGenerateClinitCheck()) {
520    CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>();
521    InvokeRuntime(kQuickInitializeStaticStorage, cls, cls->GetDexPc());
522  } else {
523    CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>();
524    InvokeRuntime(kQuickInitializeType, cls, cls->GetDexPc());
525  }
526}
527
528void CodeGenerator::BlockIfInRegister(Location location, bool is_out) const {
529  // The DCHECKS below check that a register is not specified twice in
530  // the summary. The out location can overlap with an input, so we need
531  // to special case it.
532  if (location.IsRegister()) {
533    DCHECK(is_out || !blocked_core_registers_[location.reg()]);
534    blocked_core_registers_[location.reg()] = true;
535  } else if (location.IsFpuRegister()) {
536    DCHECK(is_out || !blocked_fpu_registers_[location.reg()]);
537    blocked_fpu_registers_[location.reg()] = true;
538  } else if (location.IsFpuRegisterPair()) {
539    DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()]);
540    blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()] = true;
541    DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()]);
542    blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()] = true;
543  } else if (location.IsRegisterPair()) {
544    DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairLow<int>()]);
545    blocked_core_registers_[location.AsRegisterPairLow<int>()] = true;
546    DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairHigh<int>()]);
547    blocked_core_registers_[location.AsRegisterPairHigh<int>()] = true;
548  }
549}
550
551void CodeGenerator::AllocateLocations(HInstruction* instruction) {
552  instruction->Accept(GetLocationBuilder());
553  DCHECK(CheckTypeConsistency(instruction));
554  LocationSummary* locations = instruction->GetLocations();
555  if (!instruction->IsSuspendCheckEntry()) {
556    if (locations != nullptr) {
557      if (locations->CanCall()) {
558        MarkNotLeaf();
559      } else if (locations->Intrinsified() &&
560                 instruction->IsInvokeStaticOrDirect() &&
561                 !instruction->AsInvokeStaticOrDirect()->HasCurrentMethodInput()) {
562        // A static method call that has been fully intrinsified, and cannot call on the slow
563        // path or refer to the current method directly, no longer needs current method.
564        return;
565      }
566    }
567    if (instruction->NeedsCurrentMethod()) {
568      SetRequiresCurrentMethod();
569    }
570  }
571}
572
573void CodeGenerator::MaybeRecordStat(MethodCompilationStat compilation_stat, size_t count) const {
574  if (stats_ != nullptr) {
575    stats_->RecordStat(compilation_stat, count);
576  }
577}
578
579std::unique_ptr<CodeGenerator> CodeGenerator::Create(HGraph* graph,
580                                                     InstructionSet instruction_set,
581                                                     const InstructionSetFeatures& isa_features,
582                                                     const CompilerOptions& compiler_options,
583                                                     OptimizingCompilerStats* stats) {
584  ArenaAllocator* arena = graph->GetArena();
585  switch (instruction_set) {
586#ifdef ART_ENABLE_CODEGEN_arm
587    case kArm:
588    case kThumb2: {
589      if (kArmUseVIXL32) {
590        return std::unique_ptr<CodeGenerator>(
591            new (arena) arm::CodeGeneratorARMVIXL(graph,
592                                                  *isa_features.AsArmInstructionSetFeatures(),
593                                                  compiler_options,
594                                                  stats));
595      } else {
596          return std::unique_ptr<CodeGenerator>(
597            new (arena) arm::CodeGeneratorARM(graph,
598                                              *isa_features.AsArmInstructionSetFeatures(),
599                                              compiler_options,
600                                              stats));
601      }
602    }
603#endif
604#ifdef ART_ENABLE_CODEGEN_arm64
605    case kArm64: {
606      return std::unique_ptr<CodeGenerator>(
607          new (arena) arm64::CodeGeneratorARM64(graph,
608                                                *isa_features.AsArm64InstructionSetFeatures(),
609                                                compiler_options,
610                                                stats));
611    }
612#endif
613#ifdef ART_ENABLE_CODEGEN_mips
614    case kMips: {
615      return std::unique_ptr<CodeGenerator>(
616          new (arena) mips::CodeGeneratorMIPS(graph,
617                                              *isa_features.AsMipsInstructionSetFeatures(),
618                                              compiler_options,
619                                              stats));
620    }
621#endif
622#ifdef ART_ENABLE_CODEGEN_mips64
623    case kMips64: {
624      return std::unique_ptr<CodeGenerator>(
625          new (arena) mips64::CodeGeneratorMIPS64(graph,
626                                                  *isa_features.AsMips64InstructionSetFeatures(),
627                                                  compiler_options,
628                                                  stats));
629    }
630#endif
631#ifdef ART_ENABLE_CODEGEN_x86
632    case kX86: {
633      return std::unique_ptr<CodeGenerator>(
634          new (arena) x86::CodeGeneratorX86(graph,
635                                            *isa_features.AsX86InstructionSetFeatures(),
636                                            compiler_options,
637                                            stats));
638    }
639#endif
640#ifdef ART_ENABLE_CODEGEN_x86_64
641    case kX86_64: {
642      return std::unique_ptr<CodeGenerator>(
643          new (arena) x86_64::CodeGeneratorX86_64(graph,
644                                                  *isa_features.AsX86_64InstructionSetFeatures(),
645                                                  compiler_options,
646                                                  stats));
647    }
648#endif
649    default:
650      return nullptr;
651  }
652}
653
654size_t CodeGenerator::ComputeStackMapsSize() {
655  return stack_map_stream_.PrepareForFillIn();
656}
657
658static void CheckCovers(uint32_t dex_pc,
659                        const HGraph& graph,
660                        const CodeInfo& code_info,
661                        const ArenaVector<HSuspendCheck*>& loop_headers,
662                        ArenaVector<size_t>* covered) {
663  CodeInfoEncoding encoding = code_info.ExtractEncoding();
664  for (size_t i = 0; i < loop_headers.size(); ++i) {
665    if (loop_headers[i]->GetDexPc() == dex_pc) {
666      if (graph.IsCompilingOsr()) {
667        DCHECK(code_info.GetOsrStackMapForDexPc(dex_pc, encoding).IsValid());
668      }
669      ++(*covered)[i];
670    }
671  }
672}
673
674// Debug helper to ensure loop entries in compiled code are matched by
675// dex branch instructions.
676static void CheckLoopEntriesCanBeUsedForOsr(const HGraph& graph,
677                                            const CodeInfo& code_info,
678                                            const DexFile::CodeItem& code_item) {
679  if (graph.HasTryCatch()) {
680    // One can write loops through try/catch, which we do not support for OSR anyway.
681    return;
682  }
683  ArenaVector<HSuspendCheck*> loop_headers(graph.GetArena()->Adapter(kArenaAllocMisc));
684  for (HBasicBlock* block : graph.GetReversePostOrder()) {
685    if (block->IsLoopHeader()) {
686      HSuspendCheck* suspend_check = block->GetLoopInformation()->GetSuspendCheck();
687      if (!suspend_check->GetEnvironment()->IsFromInlinedInvoke()) {
688        loop_headers.push_back(suspend_check);
689      }
690    }
691  }
692  ArenaVector<size_t> covered(loop_headers.size(), 0, graph.GetArena()->Adapter(kArenaAllocMisc));
693  const uint16_t* code_ptr = code_item.insns_;
694  const uint16_t* code_end = code_item.insns_ + code_item.insns_size_in_code_units_;
695
696  size_t dex_pc = 0;
697  while (code_ptr < code_end) {
698    const Instruction& instruction = *Instruction::At(code_ptr);
699    if (instruction.IsBranch()) {
700      uint32_t target = dex_pc + instruction.GetTargetOffset();
701      CheckCovers(target, graph, code_info, loop_headers, &covered);
702    } else if (instruction.IsSwitch()) {
703      DexSwitchTable table(instruction, dex_pc);
704      uint16_t num_entries = table.GetNumEntries();
705      size_t offset = table.GetFirstValueIndex();
706
707      // Use a larger loop counter type to avoid overflow issues.
708      for (size_t i = 0; i < num_entries; ++i) {
709        // The target of the case.
710        uint32_t target = dex_pc + table.GetEntryAt(i + offset);
711        CheckCovers(target, graph, code_info, loop_headers, &covered);
712      }
713    }
714    dex_pc += instruction.SizeInCodeUnits();
715    code_ptr += instruction.SizeInCodeUnits();
716  }
717
718  for (size_t i = 0; i < covered.size(); ++i) {
719    DCHECK_NE(covered[i], 0u) << "Loop in compiled code has no dex branch equivalent";
720  }
721}
722
723void CodeGenerator::BuildStackMaps(MemoryRegion region, const DexFile::CodeItem& code_item) {
724  stack_map_stream_.FillIn(region);
725  if (kIsDebugBuild) {
726    CheckLoopEntriesCanBeUsedForOsr(*graph_, CodeInfo(region), code_item);
727  }
728}
729
730void CodeGenerator::RecordPcInfo(HInstruction* instruction,
731                                 uint32_t dex_pc,
732                                 SlowPathCode* slow_path) {
733  if (instruction != nullptr) {
734    // The code generated for some type conversions
735    // may call the runtime, thus normally requiring a subsequent
736    // call to this method. However, the method verifier does not
737    // produce PC information for certain instructions, which are
738    // considered "atomic" (they cannot join a GC).
739    // Therefore we do not currently record PC information for such
740    // instructions.  As this may change later, we added this special
741    // case so that code generators may nevertheless call
742    // CodeGenerator::RecordPcInfo without triggering an error in
743    // CodeGenerator::BuildNativeGCMap ("Missing ref for dex pc 0x")
744    // thereafter.
745    if (instruction->IsTypeConversion()) {
746      return;
747    }
748    if (instruction->IsRem()) {
749      Primitive::Type type = instruction->AsRem()->GetResultType();
750      if ((type == Primitive::kPrimFloat) || (type == Primitive::kPrimDouble)) {
751        return;
752      }
753    }
754  }
755
756  uint32_t outer_dex_pc = dex_pc;
757  uint32_t outer_environment_size = 0;
758  uint32_t inlining_depth = 0;
759  if (instruction != nullptr) {
760    for (HEnvironment* environment = instruction->GetEnvironment();
761         environment != nullptr;
762         environment = environment->GetParent()) {
763      outer_dex_pc = environment->GetDexPc();
764      outer_environment_size = environment->Size();
765      if (environment != instruction->GetEnvironment()) {
766        inlining_depth++;
767      }
768    }
769  }
770
771  // Collect PC infos for the mapping table.
772  uint32_t native_pc = GetAssembler()->CodePosition();
773
774  if (instruction == nullptr) {
775    // For stack overflow checks and native-debug-info entries without dex register
776    // mapping (i.e. start of basic block or start of slow path).
777    stack_map_stream_.BeginStackMapEntry(outer_dex_pc, native_pc, 0, 0, 0, 0);
778    stack_map_stream_.EndStackMapEntry();
779    return;
780  }
781  LocationSummary* locations = instruction->GetLocations();
782
783  uint32_t register_mask = locations->GetRegisterMask();
784  DCHECK_EQ(register_mask & ~locations->GetLiveRegisters()->GetCoreRegisters(), 0u);
785  if (locations->OnlyCallsOnSlowPath()) {
786    // In case of slow path, we currently set the location of caller-save registers
787    // to register (instead of their stack location when pushed before the slow-path
788    // call). Therefore register_mask contains both callee-save and caller-save
789    // registers that hold objects. We must remove the spilled caller-save from the
790    // mask, since they will be overwritten by the callee.
791    uint32_t spills = GetSlowPathSpills(locations, /* core_registers */ true);
792    register_mask &= ~spills;
793  } else {
794    // The register mask must be a subset of callee-save registers.
795    DCHECK_EQ(register_mask & core_callee_save_mask_, register_mask);
796  }
797  stack_map_stream_.BeginStackMapEntry(outer_dex_pc,
798                                       native_pc,
799                                       register_mask,
800                                       locations->GetStackMask(),
801                                       outer_environment_size,
802                                       inlining_depth);
803
804  EmitEnvironment(instruction->GetEnvironment(), slow_path);
805  stack_map_stream_.EndStackMapEntry();
806
807  HLoopInformation* info = instruction->GetBlock()->GetLoopInformation();
808  if (instruction->IsSuspendCheck() &&
809      (info != nullptr) &&
810      graph_->IsCompilingOsr() &&
811      (inlining_depth == 0)) {
812    DCHECK_EQ(info->GetSuspendCheck(), instruction);
813    // We duplicate the stack map as a marker that this stack map can be an OSR entry.
814    // Duplicating it avoids having the runtime recognize and skip an OSR stack map.
815    DCHECK(info->IsIrreducible());
816    stack_map_stream_.BeginStackMapEntry(
817        dex_pc, native_pc, register_mask, locations->GetStackMask(), outer_environment_size, 0);
818    EmitEnvironment(instruction->GetEnvironment(), slow_path);
819    stack_map_stream_.EndStackMapEntry();
820    if (kIsDebugBuild) {
821      HEnvironment* environment = instruction->GetEnvironment();
822      for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) {
823        HInstruction* in_environment = environment->GetInstructionAt(i);
824        if (in_environment != nullptr) {
825          DCHECK(in_environment->IsPhi() || in_environment->IsConstant());
826          Location location = environment->GetLocationAt(i);
827          DCHECK(location.IsStackSlot() ||
828                 location.IsDoubleStackSlot() ||
829                 location.IsConstant() ||
830                 location.IsInvalid());
831          if (location.IsStackSlot() || location.IsDoubleStackSlot()) {
832            DCHECK_LT(location.GetStackIndex(), static_cast<int32_t>(GetFrameSize()));
833          }
834        }
835      }
836    }
837  } else if (kIsDebugBuild) {
838    // Ensure stack maps are unique, by checking that the native pc in the stack map
839    // last emitted is different than the native pc of the stack map just emitted.
840    size_t number_of_stack_maps = stack_map_stream_.GetNumberOfStackMaps();
841    if (number_of_stack_maps > 1) {
842      DCHECK_NE(stack_map_stream_.GetStackMap(number_of_stack_maps - 1).native_pc_code_offset,
843                stack_map_stream_.GetStackMap(number_of_stack_maps - 2).native_pc_code_offset);
844    }
845  }
846}
847
848bool CodeGenerator::HasStackMapAtCurrentPc() {
849  uint32_t pc = GetAssembler()->CodeSize();
850  size_t count = stack_map_stream_.GetNumberOfStackMaps();
851  CodeOffset native_pc_offset = stack_map_stream_.GetStackMap(count - 1).native_pc_code_offset;
852  return (count > 0) && (native_pc_offset.Uint32Value(GetInstructionSet()) == pc);
853}
854
855void CodeGenerator::MaybeRecordNativeDebugInfo(HInstruction* instruction,
856                                               uint32_t dex_pc,
857                                               SlowPathCode* slow_path) {
858  if (GetCompilerOptions().GetNativeDebuggable() && dex_pc != kNoDexPc) {
859    if (HasStackMapAtCurrentPc()) {
860      // Ensure that we do not collide with the stack map of the previous instruction.
861      GenerateNop();
862    }
863    RecordPcInfo(instruction, dex_pc, slow_path);
864  }
865}
866
867void CodeGenerator::RecordCatchBlockInfo() {
868  ArenaAllocator* arena = graph_->GetArena();
869
870  for (HBasicBlock* block : *block_order_) {
871    if (!block->IsCatchBlock()) {
872      continue;
873    }
874
875    uint32_t dex_pc = block->GetDexPc();
876    uint32_t num_vregs = graph_->GetNumberOfVRegs();
877    uint32_t inlining_depth = 0;  // Inlining of catch blocks is not supported at the moment.
878    uint32_t native_pc = GetAddressOf(block);
879    uint32_t register_mask = 0;   // Not used.
880
881    // The stack mask is not used, so we leave it empty.
882    ArenaBitVector* stack_mask =
883        ArenaBitVector::Create(arena, 0, /* expandable */ true, kArenaAllocCodeGenerator);
884
885    stack_map_stream_.BeginStackMapEntry(dex_pc,
886                                         native_pc,
887                                         register_mask,
888                                         stack_mask,
889                                         num_vregs,
890                                         inlining_depth);
891
892    HInstruction* current_phi = block->GetFirstPhi();
893    for (size_t vreg = 0; vreg < num_vregs; ++vreg) {
894    while (current_phi != nullptr && current_phi->AsPhi()->GetRegNumber() < vreg) {
895      HInstruction* next_phi = current_phi->GetNext();
896      DCHECK(next_phi == nullptr ||
897             current_phi->AsPhi()->GetRegNumber() <= next_phi->AsPhi()->GetRegNumber())
898          << "Phis need to be sorted by vreg number to keep this a linear-time loop.";
899      current_phi = next_phi;
900    }
901
902      if (current_phi == nullptr || current_phi->AsPhi()->GetRegNumber() != vreg) {
903        stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
904      } else {
905        Location location = current_phi->GetLiveInterval()->ToLocation();
906        switch (location.GetKind()) {
907          case Location::kStackSlot: {
908            stack_map_stream_.AddDexRegisterEntry(
909                DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
910            break;
911          }
912          case Location::kDoubleStackSlot: {
913            stack_map_stream_.AddDexRegisterEntry(
914                DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
915            stack_map_stream_.AddDexRegisterEntry(
916                DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize));
917            ++vreg;
918            DCHECK_LT(vreg, num_vregs);
919            break;
920          }
921          default: {
922            // All catch phis must be allocated to a stack slot.
923            LOG(FATAL) << "Unexpected kind " << location.GetKind();
924            UNREACHABLE();
925          }
926        }
927      }
928    }
929
930    stack_map_stream_.EndStackMapEntry();
931  }
932}
933
934void CodeGenerator::EmitEnvironment(HEnvironment* environment, SlowPathCode* slow_path) {
935  if (environment == nullptr) return;
936
937  if (environment->GetParent() != nullptr) {
938    // We emit the parent environment first.
939    EmitEnvironment(environment->GetParent(), slow_path);
940    stack_map_stream_.BeginInlineInfoEntry(environment->GetMethod(),
941                                           environment->GetDexPc(),
942                                           environment->Size(),
943                                           &graph_->GetDexFile());
944  }
945
946  // Walk over the environment, and record the location of dex registers.
947  for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) {
948    HInstruction* current = environment->GetInstructionAt(i);
949    if (current == nullptr) {
950      stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
951      continue;
952    }
953
954    Location location = environment->GetLocationAt(i);
955    switch (location.GetKind()) {
956      case Location::kConstant: {
957        DCHECK_EQ(current, location.GetConstant());
958        if (current->IsLongConstant()) {
959          int64_t value = current->AsLongConstant()->GetValue();
960          stack_map_stream_.AddDexRegisterEntry(
961              DexRegisterLocation::Kind::kConstant, Low32Bits(value));
962          stack_map_stream_.AddDexRegisterEntry(
963              DexRegisterLocation::Kind::kConstant, High32Bits(value));
964          ++i;
965          DCHECK_LT(i, environment_size);
966        } else if (current->IsDoubleConstant()) {
967          int64_t value = bit_cast<int64_t, double>(current->AsDoubleConstant()->GetValue());
968          stack_map_stream_.AddDexRegisterEntry(
969              DexRegisterLocation::Kind::kConstant, Low32Bits(value));
970          stack_map_stream_.AddDexRegisterEntry(
971              DexRegisterLocation::Kind::kConstant, High32Bits(value));
972          ++i;
973          DCHECK_LT(i, environment_size);
974        } else if (current->IsIntConstant()) {
975          int32_t value = current->AsIntConstant()->GetValue();
976          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, value);
977        } else if (current->IsNullConstant()) {
978          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, 0);
979        } else {
980          DCHECK(current->IsFloatConstant()) << current->DebugName();
981          int32_t value = bit_cast<int32_t, float>(current->AsFloatConstant()->GetValue());
982          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, value);
983        }
984        break;
985      }
986
987      case Location::kStackSlot: {
988        stack_map_stream_.AddDexRegisterEntry(
989            DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
990        break;
991      }
992
993      case Location::kDoubleStackSlot: {
994        stack_map_stream_.AddDexRegisterEntry(
995            DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
996        stack_map_stream_.AddDexRegisterEntry(
997            DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize));
998        ++i;
999        DCHECK_LT(i, environment_size);
1000        break;
1001      }
1002
1003      case Location::kRegister : {
1004        int id = location.reg();
1005        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(id)) {
1006          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(id);
1007          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1008          if (current->GetType() == Primitive::kPrimLong) {
1009            stack_map_stream_.AddDexRegisterEntry(
1010                DexRegisterLocation::Kind::kInStack, offset + kVRegSize);
1011            ++i;
1012            DCHECK_LT(i, environment_size);
1013          }
1014        } else {
1015          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, id);
1016          if (current->GetType() == Primitive::kPrimLong) {
1017            stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegisterHigh, id);
1018            ++i;
1019            DCHECK_LT(i, environment_size);
1020          }
1021        }
1022        break;
1023      }
1024
1025      case Location::kFpuRegister : {
1026        int id = location.reg();
1027        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(id)) {
1028          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(id);
1029          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1030          if (current->GetType() == Primitive::kPrimDouble) {
1031            stack_map_stream_.AddDexRegisterEntry(
1032                DexRegisterLocation::Kind::kInStack, offset + kVRegSize);
1033            ++i;
1034            DCHECK_LT(i, environment_size);
1035          }
1036        } else {
1037          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, id);
1038          if (current->GetType() == Primitive::kPrimDouble) {
1039            stack_map_stream_.AddDexRegisterEntry(
1040                DexRegisterLocation::Kind::kInFpuRegisterHigh, id);
1041            ++i;
1042            DCHECK_LT(i, environment_size);
1043          }
1044        }
1045        break;
1046      }
1047
1048      case Location::kFpuRegisterPair : {
1049        int low = location.low();
1050        int high = location.high();
1051        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(low)) {
1052          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(low);
1053          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1054        } else {
1055          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, low);
1056        }
1057        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(high)) {
1058          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(high);
1059          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1060          ++i;
1061        } else {
1062          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, high);
1063          ++i;
1064        }
1065        DCHECK_LT(i, environment_size);
1066        break;
1067      }
1068
1069      case Location::kRegisterPair : {
1070        int low = location.low();
1071        int high = location.high();
1072        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(low)) {
1073          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(low);
1074          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1075        } else {
1076          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, low);
1077        }
1078        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(high)) {
1079          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(high);
1080          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1081        } else {
1082          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, high);
1083        }
1084        ++i;
1085        DCHECK_LT(i, environment_size);
1086        break;
1087      }
1088
1089      case Location::kInvalid: {
1090        stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
1091        break;
1092      }
1093
1094      default:
1095        LOG(FATAL) << "Unexpected kind " << location.GetKind();
1096    }
1097  }
1098
1099  if (environment->GetParent() != nullptr) {
1100    stack_map_stream_.EndInlineInfoEntry();
1101  }
1102}
1103
1104bool CodeGenerator::CanMoveNullCheckToUser(HNullCheck* null_check) {
1105  HInstruction* first_next_not_move = null_check->GetNextDisregardingMoves();
1106
1107  return (first_next_not_move != nullptr)
1108      && first_next_not_move->CanDoImplicitNullCheckOn(null_check->InputAt(0));
1109}
1110
1111void CodeGenerator::MaybeRecordImplicitNullCheck(HInstruction* instr) {
1112  if (!compiler_options_.GetImplicitNullChecks()) {
1113    return;
1114  }
1115
1116  // If we are from a static path don't record the pc as we can't throw NPE.
1117  // NB: having the checks here makes the code much less verbose in the arch
1118  // specific code generators.
1119  if (instr->IsStaticFieldSet() || instr->IsStaticFieldGet()) {
1120    return;
1121  }
1122
1123  if (!instr->CanDoImplicitNullCheckOn(instr->InputAt(0))) {
1124    return;
1125  }
1126
1127  // Find the first previous instruction which is not a move.
1128  HInstruction* first_prev_not_move = instr->GetPreviousDisregardingMoves();
1129
1130  // If the instruction is a null check it means that `instr` is the first user
1131  // and needs to record the pc.
1132  if (first_prev_not_move != nullptr && first_prev_not_move->IsNullCheck()) {
1133    HNullCheck* null_check = first_prev_not_move->AsNullCheck();
1134    // TODO: The parallel moves modify the environment. Their changes need to be
1135    // reverted otherwise the stack maps at the throw point will not be correct.
1136    RecordPcInfo(null_check, null_check->GetDexPc());
1137  }
1138}
1139
1140LocationSummary* CodeGenerator::CreateThrowingSlowPathLocations(HInstruction* instruction,
1141                                                                RegisterSet caller_saves) {
1142  // Note: Using kNoCall allows the method to be treated as leaf (and eliminate the
1143  // HSuspendCheck from entry block). However, it will still get a valid stack frame
1144  // because the HNullCheck needs an environment.
1145  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
1146  // When throwing from a try block, we may need to retrieve dalvik registers from
1147  // physical registers and we also need to set up stack mask for GC. This is
1148  // implicitly achieved by passing kCallOnSlowPath to the LocationSummary.
1149  bool can_throw_into_catch_block = instruction->CanThrowIntoCatchBlock();
1150  if (can_throw_into_catch_block) {
1151    call_kind = LocationSummary::kCallOnSlowPath;
1152  }
1153  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
1154  if (can_throw_into_catch_block && compiler_options_.GetImplicitNullChecks()) {
1155    locations->SetCustomSlowPathCallerSaves(caller_saves);  // Default: no caller-save registers.
1156  }
1157  DCHECK(!instruction->HasUses());
1158  return locations;
1159}
1160
1161void CodeGenerator::GenerateNullCheck(HNullCheck* instruction) {
1162  if (compiler_options_.GetImplicitNullChecks()) {
1163    MaybeRecordStat(kImplicitNullCheckGenerated);
1164    GenerateImplicitNullCheck(instruction);
1165  } else {
1166    MaybeRecordStat(kExplicitNullCheckGenerated);
1167    GenerateExplicitNullCheck(instruction);
1168  }
1169}
1170
1171void CodeGenerator::ClearSpillSlotsFromLoopPhisInStackMap(HSuspendCheck* suspend_check) const {
1172  LocationSummary* locations = suspend_check->GetLocations();
1173  HBasicBlock* block = suspend_check->GetBlock();
1174  DCHECK(block->GetLoopInformation()->GetSuspendCheck() == suspend_check);
1175  DCHECK(block->IsLoopHeader());
1176
1177  for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) {
1178    HInstruction* current = it.Current();
1179    LiveInterval* interval = current->GetLiveInterval();
1180    // We only need to clear bits of loop phis containing objects and allocated in register.
1181    // Loop phis allocated on stack already have the object in the stack.
1182    if (current->GetType() == Primitive::kPrimNot
1183        && interval->HasRegister()
1184        && interval->HasSpillSlot()) {
1185      locations->ClearStackBit(interval->GetSpillSlot() / kVRegSize);
1186    }
1187  }
1188}
1189
1190void CodeGenerator::EmitParallelMoves(Location from1,
1191                                      Location to1,
1192                                      Primitive::Type type1,
1193                                      Location from2,
1194                                      Location to2,
1195                                      Primitive::Type type2) {
1196  HParallelMove parallel_move(GetGraph()->GetArena());
1197  parallel_move.AddMove(from1, to1, type1, nullptr);
1198  parallel_move.AddMove(from2, to2, type2, nullptr);
1199  GetMoveResolver()->EmitNativeCode(&parallel_move);
1200}
1201
1202void CodeGenerator::ValidateInvokeRuntime(QuickEntrypointEnum entrypoint,
1203                                          HInstruction* instruction,
1204                                          SlowPathCode* slow_path) {
1205  // Ensure that the call kind indication given to the register allocator is
1206  // coherent with the runtime call generated.
1207  if (slow_path == nullptr) {
1208    DCHECK(instruction->GetLocations()->WillCall())
1209        << "instruction->DebugName()=" << instruction->DebugName();
1210  } else {
1211    DCHECK(instruction->GetLocations()->CallsOnSlowPath() || slow_path->IsFatal())
1212        << "instruction->DebugName()=" << instruction->DebugName()
1213        << " slow_path->GetDescription()=" << slow_path->GetDescription();
1214  }
1215
1216  // Check that the GC side effect is set when required.
1217  // TODO: Reverse EntrypointCanTriggerGC
1218  if (EntrypointCanTriggerGC(entrypoint)) {
1219    if (slow_path == nullptr) {
1220      DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()))
1221          << "instruction->DebugName()=" << instruction->DebugName()
1222          << " instruction->GetSideEffects().ToString()="
1223          << instruction->GetSideEffects().ToString();
1224    } else {
1225      DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()) ||
1226             // When (non-Baker) read barriers are enabled, some instructions
1227             // use a slow path to emit a read barrier, which does not trigger
1228             // GC.
1229             (kEmitCompilerReadBarrier &&
1230              !kUseBakerReadBarrier &&
1231              (instruction->IsInstanceFieldGet() ||
1232               instruction->IsStaticFieldGet() ||
1233               instruction->IsArrayGet() ||
1234               instruction->IsLoadClass() ||
1235               instruction->IsLoadString() ||
1236               instruction->IsInstanceOf() ||
1237               instruction->IsCheckCast() ||
1238               (instruction->IsInvokeVirtual() && instruction->GetLocations()->Intrinsified()))))
1239          << "instruction->DebugName()=" << instruction->DebugName()
1240          << " instruction->GetSideEffects().ToString()="
1241          << instruction->GetSideEffects().ToString()
1242          << " slow_path->GetDescription()=" << slow_path->GetDescription();
1243    }
1244  } else {
1245    // The GC side effect is not required for the instruction. But the instruction might still have
1246    // it, for example if it calls other entrypoints requiring it.
1247  }
1248
1249  // Check the coherency of leaf information.
1250  DCHECK(instruction->IsSuspendCheck()
1251         || ((slow_path != nullptr) && slow_path->IsFatal())
1252         || instruction->GetLocations()->CanCall()
1253         || !IsLeafMethod())
1254      << instruction->DebugName() << ((slow_path != nullptr) ? slow_path->GetDescription() : "");
1255}
1256
1257void CodeGenerator::ValidateInvokeRuntimeWithoutRecordingPcInfo(HInstruction* instruction,
1258                                                                SlowPathCode* slow_path) {
1259  DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath())
1260      << "instruction->DebugName()=" << instruction->DebugName()
1261      << " slow_path->GetDescription()=" << slow_path->GetDescription();
1262  // Only the Baker read barrier marking slow path used by certains
1263  // instructions is expected to invoke the runtime without recording
1264  // PC-related information.
1265  DCHECK(kUseBakerReadBarrier);
1266  DCHECK(instruction->IsInstanceFieldGet() ||
1267         instruction->IsStaticFieldGet() ||
1268         instruction->IsArrayGet() ||
1269         instruction->IsArraySet() ||
1270         instruction->IsLoadClass() ||
1271         instruction->IsLoadString() ||
1272         instruction->IsInstanceOf() ||
1273         instruction->IsCheckCast() ||
1274         (instruction->IsInvokeVirtual() && instruction->GetLocations()->Intrinsified()) ||
1275         (instruction->IsInvokeStaticOrDirect() && instruction->GetLocations()->Intrinsified()))
1276      << "instruction->DebugName()=" << instruction->DebugName()
1277      << " slow_path->GetDescription()=" << slow_path->GetDescription();
1278}
1279
1280void SlowPathCode::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
1281  size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
1282
1283  const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true);
1284  for (uint32_t i : LowToHighBits(core_spills)) {
1285    // If the register holds an object, update the stack mask.
1286    if (locations->RegisterContainsObject(i)) {
1287      locations->SetStackBit(stack_offset / kVRegSize);
1288    }
1289    DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1290    DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1291    saved_core_stack_offsets_[i] = stack_offset;
1292    stack_offset += codegen->SaveCoreRegister(stack_offset, i);
1293  }
1294
1295  const uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false);
1296  for (uint32_t i : LowToHighBits(fp_spills)) {
1297    DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1298    DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1299    saved_fpu_stack_offsets_[i] = stack_offset;
1300    stack_offset += codegen->SaveFloatingPointRegister(stack_offset, i);
1301  }
1302}
1303
1304void SlowPathCode::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
1305  size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
1306
1307  const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true);
1308  for (uint32_t i : LowToHighBits(core_spills)) {
1309    DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1310    DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1311    stack_offset += codegen->RestoreCoreRegister(stack_offset, i);
1312  }
1313
1314  const uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false);
1315  for (uint32_t i : LowToHighBits(fp_spills)) {
1316    DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1317    DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1318    stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, i);
1319  }
1320}
1321
1322void CodeGenerator::CreateSystemArrayCopyLocationSummary(HInvoke* invoke) {
1323  // Check to see if we have known failures that will cause us to have to bail out
1324  // to the runtime, and just generate the runtime call directly.
1325  HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant();
1326  HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant();
1327
1328  // The positions must be non-negative.
1329  if ((src_pos != nullptr && src_pos->GetValue() < 0) ||
1330      (dest_pos != nullptr && dest_pos->GetValue() < 0)) {
1331    // We will have to fail anyways.
1332    return;
1333  }
1334
1335  // The length must be >= 0.
1336  HIntConstant* length = invoke->InputAt(4)->AsIntConstant();
1337  if (length != nullptr) {
1338    int32_t len = length->GetValue();
1339    if (len < 0) {
1340      // Just call as normal.
1341      return;
1342    }
1343  }
1344
1345  SystemArrayCopyOptimizations optimizations(invoke);
1346
1347  if (optimizations.GetDestinationIsSource()) {
1348    if (src_pos != nullptr && dest_pos != nullptr && src_pos->GetValue() < dest_pos->GetValue()) {
1349      // We only support backward copying if source and destination are the same.
1350      return;
1351    }
1352  }
1353
1354  if (optimizations.GetDestinationIsPrimitiveArray() || optimizations.GetSourceIsPrimitiveArray()) {
1355    // We currently don't intrinsify primitive copying.
1356    return;
1357  }
1358
1359  ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena();
1360  LocationSummary* locations = new (allocator) LocationSummary(invoke,
1361                                                               LocationSummary::kCallOnSlowPath,
1362                                                               kIntrinsified);
1363  // arraycopy(Object src, int src_pos, Object dest, int dest_pos, int length).
1364  locations->SetInAt(0, Location::RequiresRegister());
1365  locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1)));
1366  locations->SetInAt(2, Location::RequiresRegister());
1367  locations->SetInAt(3, Location::RegisterOrConstant(invoke->InputAt(3)));
1368  locations->SetInAt(4, Location::RegisterOrConstant(invoke->InputAt(4)));
1369
1370  locations->AddTemp(Location::RequiresRegister());
1371  locations->AddTemp(Location::RequiresRegister());
1372  locations->AddTemp(Location::RequiresRegister());
1373}
1374
1375uint32_t CodeGenerator::GetReferenceSlowFlagOffset() const {
1376  ScopedObjectAccess soa(Thread::Current());
1377  mirror::Class* klass = mirror::Reference::GetJavaLangRefReference();
1378  DCHECK(klass->IsInitialized());
1379  return klass->GetSlowPathFlagOffset().Uint32Value();
1380}
1381
1382uint32_t CodeGenerator::GetReferenceDisableFlagOffset() const {
1383  ScopedObjectAccess soa(Thread::Current());
1384  mirror::Class* klass = mirror::Reference::GetJavaLangRefReference();
1385  DCHECK(klass->IsInitialized());
1386  return klass->GetDisableIntrinsicFlagOffset().Uint32Value();
1387}
1388
1389void CodeGenerator::EmitJitRoots(uint8_t* code,
1390                                 Handle<mirror::ObjectArray<mirror::Object>> roots,
1391                                 const uint8_t* roots_data) {
1392  DCHECK_EQ(static_cast<size_t>(roots->GetLength()), GetNumberOfJitRoots());
1393  ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
1394  size_t index = 0;
1395  for (auto& entry : jit_string_roots_) {
1396    // Update the `roots` with the string, and replace the address temporarily
1397    // stored to the index in the table.
1398    uint64_t address = entry.second;
1399    roots->Set(index, reinterpret_cast<StackReference<mirror::String>*>(address)->AsMirrorPtr());
1400    DCHECK(roots->Get(index) != nullptr);
1401    entry.second = index;
1402    // Ensure the string is strongly interned. This is a requirement on how the JIT
1403    // handles strings. b/32995596
1404    class_linker->GetInternTable()->InternStrong(
1405        reinterpret_cast<mirror::String*>(roots->Get(index)));
1406    ++index;
1407  }
1408  for (auto& entry : jit_class_roots_) {
1409    // Update the `roots` with the class, and replace the address temporarily
1410    // stored to the index in the table.
1411    uint64_t address = entry.second;
1412    roots->Set(index, reinterpret_cast<StackReference<mirror::Class>*>(address)->AsMirrorPtr());
1413    DCHECK(roots->Get(index) != nullptr);
1414    entry.second = index;
1415    ++index;
1416  }
1417  EmitJitRootPatches(code, roots_data);
1418}
1419
1420QuickEntrypointEnum CodeGenerator::GetArrayAllocationEntrypoint(Handle<mirror::Class> array_klass) {
1421  ScopedObjectAccess soa(Thread::Current());
1422  if (array_klass == nullptr) {
1423    // This can only happen for non-primitive arrays, as primitive arrays can always
1424    // be resolved.
1425    return kQuickAllocArrayResolved32;
1426  }
1427
1428  switch (array_klass->GetComponentSize()) {
1429    case 1: return kQuickAllocArrayResolved8;
1430    case 2: return kQuickAllocArrayResolved16;
1431    case 4: return kQuickAllocArrayResolved32;
1432    case 8: return kQuickAllocArrayResolved64;
1433  }
1434  LOG(FATAL) << "Unreachable";
1435  return kQuickAllocArrayResolved;
1436}
1437
1438}  // namespace art
1439