code_generator.cc revision dce016eab87302f02b0bd903dd2cd86ae512df2d 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator.h"
18
19#ifdef ART_ENABLE_CODEGEN_arm
20#include "code_generator_arm.h"
21#endif
22
23#ifdef ART_ENABLE_CODEGEN_arm64
24#include "code_generator_arm64.h"
25#endif
26
27#ifdef ART_ENABLE_CODEGEN_x86
28#include "code_generator_x86.h"
29#endif
30
31#ifdef ART_ENABLE_CODEGEN_x86_64
32#include "code_generator_x86_64.h"
33#endif
34
35#ifdef ART_ENABLE_CODEGEN_mips
36#include "code_generator_mips.h"
37#endif
38
39#ifdef ART_ENABLE_CODEGEN_mips64
40#include "code_generator_mips64.h"
41#endif
42
43#include "bytecode_utils.h"
44#include "compiled_method.h"
45#include "dex/verified_method.h"
46#include "driver/compiler_driver.h"
47#include "graph_visualizer.h"
48#include "intrinsics.h"
49#include "leb128.h"
50#include "mirror/array-inl.h"
51#include "mirror/object_array-inl.h"
52#include "mirror/object_reference.h"
53#include "mirror/string.h"
54#include "parallel_move_resolver.h"
55#include "ssa_liveness_analysis.h"
56#include "utils/assembler.h"
57
58namespace art {
59
60// Return whether a location is consistent with a type.
61static bool CheckType(Primitive::Type type, Location location) {
62  if (location.IsFpuRegister()
63      || (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresFpuRegister))) {
64    return (type == Primitive::kPrimFloat) || (type == Primitive::kPrimDouble);
65  } else if (location.IsRegister() ||
66             (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresRegister))) {
67    return Primitive::IsIntegralType(type) || (type == Primitive::kPrimNot);
68  } else if (location.IsRegisterPair()) {
69    return type == Primitive::kPrimLong;
70  } else if (location.IsFpuRegisterPair()) {
71    return type == Primitive::kPrimDouble;
72  } else if (location.IsStackSlot()) {
73    return (Primitive::IsIntegralType(type) && type != Primitive::kPrimLong)
74           || (type == Primitive::kPrimFloat)
75           || (type == Primitive::kPrimNot);
76  } else if (location.IsDoubleStackSlot()) {
77    return (type == Primitive::kPrimLong) || (type == Primitive::kPrimDouble);
78  } else if (location.IsConstant()) {
79    if (location.GetConstant()->IsIntConstant()) {
80      return Primitive::IsIntegralType(type) && (type != Primitive::kPrimLong);
81    } else if (location.GetConstant()->IsNullConstant()) {
82      return type == Primitive::kPrimNot;
83    } else if (location.GetConstant()->IsLongConstant()) {
84      return type == Primitive::kPrimLong;
85    } else if (location.GetConstant()->IsFloatConstant()) {
86      return type == Primitive::kPrimFloat;
87    } else {
88      return location.GetConstant()->IsDoubleConstant()
89          && (type == Primitive::kPrimDouble);
90    }
91  } else {
92    return location.IsInvalid() || (location.GetPolicy() == Location::kAny);
93  }
94}
95
96// Check that a location summary is consistent with an instruction.
97static bool CheckTypeConsistency(HInstruction* instruction) {
98  LocationSummary* locations = instruction->GetLocations();
99  if (locations == nullptr) {
100    return true;
101  }
102
103  if (locations->Out().IsUnallocated()
104      && (locations->Out().GetPolicy() == Location::kSameAsFirstInput)) {
105    DCHECK(CheckType(instruction->GetType(), locations->InAt(0)))
106        << instruction->GetType()
107        << " " << locations->InAt(0);
108  } else {
109    DCHECK(CheckType(instruction->GetType(), locations->Out()))
110        << instruction->GetType()
111        << " " << locations->Out();
112  }
113
114  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
115    DCHECK(CheckType(instruction->InputAt(i)->GetType(), locations->InAt(i)))
116      << instruction->InputAt(i)->GetType()
117      << " " << locations->InAt(i);
118  }
119
120  HEnvironment* environment = instruction->GetEnvironment();
121  for (size_t i = 0; i < instruction->EnvironmentSize(); ++i) {
122    if (environment->GetInstructionAt(i) != nullptr) {
123      Primitive::Type type = environment->GetInstructionAt(i)->GetType();
124      DCHECK(CheckType(type, environment->GetLocationAt(i)))
125        << type << " " << environment->GetLocationAt(i);
126    } else {
127      DCHECK(environment->GetLocationAt(i).IsInvalid())
128        << environment->GetLocationAt(i);
129    }
130  }
131  return true;
132}
133
134size_t CodeGenerator::GetCacheOffset(uint32_t index) {
135  return sizeof(GcRoot<mirror::Object>) * index;
136}
137
138size_t CodeGenerator::GetCachePointerOffset(uint32_t index) {
139  auto pointer_size = InstructionSetPointerSize(GetInstructionSet());
140  return pointer_size * index;
141}
142
143uint32_t CodeGenerator::GetArrayLengthOffset(HArrayLength* array_length) {
144  return array_length->IsStringLength()
145      ? mirror::String::CountOffset().Uint32Value()
146      : mirror::Array::LengthOffset().Uint32Value();
147}
148
149bool CodeGenerator::GoesToNextBlock(HBasicBlock* current, HBasicBlock* next) const {
150  DCHECK_EQ((*block_order_)[current_block_index_], current);
151  return GetNextBlockToEmit() == FirstNonEmptyBlock(next);
152}
153
154HBasicBlock* CodeGenerator::GetNextBlockToEmit() const {
155  for (size_t i = current_block_index_ + 1; i < block_order_->size(); ++i) {
156    HBasicBlock* block = (*block_order_)[i];
157    if (!block->IsSingleJump()) {
158      return block;
159    }
160  }
161  return nullptr;
162}
163
164HBasicBlock* CodeGenerator::FirstNonEmptyBlock(HBasicBlock* block) const {
165  while (block->IsSingleJump()) {
166    block = block->GetSuccessors()[0];
167  }
168  return block;
169}
170
171class DisassemblyScope {
172 public:
173  DisassemblyScope(HInstruction* instruction, const CodeGenerator& codegen)
174      : codegen_(codegen), instruction_(instruction), start_offset_(static_cast<size_t>(-1)) {
175    if (codegen_.GetDisassemblyInformation() != nullptr) {
176      start_offset_ = codegen_.GetAssembler().CodeSize();
177    }
178  }
179
180  ~DisassemblyScope() {
181    // We avoid building this data when we know it will not be used.
182    if (codegen_.GetDisassemblyInformation() != nullptr) {
183      codegen_.GetDisassemblyInformation()->AddInstructionInterval(
184          instruction_, start_offset_, codegen_.GetAssembler().CodeSize());
185    }
186  }
187
188 private:
189  const CodeGenerator& codegen_;
190  HInstruction* instruction_;
191  size_t start_offset_;
192};
193
194
195void CodeGenerator::GenerateSlowPaths() {
196  size_t code_start = 0;
197  for (const std::unique_ptr<SlowPathCode>& slow_path_unique_ptr : slow_paths_) {
198    SlowPathCode* slow_path = slow_path_unique_ptr.get();
199    current_slow_path_ = slow_path;
200    if (disasm_info_ != nullptr) {
201      code_start = GetAssembler()->CodeSize();
202    }
203    // Record the dex pc at start of slow path (required for java line number mapping).
204    MaybeRecordNativeDebugInfo(slow_path->GetInstruction(), slow_path->GetDexPc(), slow_path);
205    slow_path->EmitNativeCode(this);
206    if (disasm_info_ != nullptr) {
207      disasm_info_->AddSlowPathInterval(slow_path, code_start, GetAssembler()->CodeSize());
208    }
209  }
210  current_slow_path_ = nullptr;
211}
212
213void CodeGenerator::Compile(CodeAllocator* allocator) {
214  // The register allocator already called `InitializeCodeGeneration`,
215  // where the frame size has been computed.
216  DCHECK(block_order_ != nullptr);
217  Initialize();
218
219  HGraphVisitor* instruction_visitor = GetInstructionVisitor();
220  DCHECK_EQ(current_block_index_, 0u);
221
222  size_t frame_start = GetAssembler()->CodeSize();
223  GenerateFrameEntry();
224  DCHECK_EQ(GetAssembler()->cfi().GetCurrentCFAOffset(), static_cast<int>(frame_size_));
225  if (disasm_info_ != nullptr) {
226    disasm_info_->SetFrameEntryInterval(frame_start, GetAssembler()->CodeSize());
227  }
228
229  for (size_t e = block_order_->size(); current_block_index_ < e; ++current_block_index_) {
230    HBasicBlock* block = (*block_order_)[current_block_index_];
231    // Don't generate code for an empty block. Its predecessors will branch to its successor
232    // directly. Also, the label of that block will not be emitted, so this helps catch
233    // errors where we reference that label.
234    if (block->IsSingleJump()) continue;
235    Bind(block);
236    // This ensures that we have correct native line mapping for all native instructions.
237    // It is necessary to make stepping over a statement work. Otherwise, any initial
238    // instructions (e.g. moves) would be assumed to be the start of next statement.
239    MaybeRecordNativeDebugInfo(nullptr /* instruction */, block->GetDexPc());
240    for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) {
241      HInstruction* current = it.Current();
242      if (current->HasEnvironment()) {
243        // Create stackmap for HNativeDebugInfo or any instruction which calls native code.
244        // Note that we need correct mapping for the native PC of the call instruction,
245        // so the runtime's stackmap is not sufficient since it is at PC after the call.
246        MaybeRecordNativeDebugInfo(current, block->GetDexPc());
247      }
248      DisassemblyScope disassembly_scope(current, *this);
249      DCHECK(CheckTypeConsistency(current));
250      current->Accept(instruction_visitor);
251    }
252  }
253
254  GenerateSlowPaths();
255
256  // Emit catch stack maps at the end of the stack map stream as expected by the
257  // runtime exception handler.
258  if (graph_->HasTryCatch()) {
259    RecordCatchBlockInfo();
260  }
261
262  // Finalize instructions in assember;
263  Finalize(allocator);
264}
265
266void CodeGenerator::Finalize(CodeAllocator* allocator) {
267  size_t code_size = GetAssembler()->CodeSize();
268  uint8_t* buffer = allocator->Allocate(code_size);
269
270  MemoryRegion code(buffer, code_size);
271  GetAssembler()->FinalizeInstructions(code);
272}
273
274void CodeGenerator::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches ATTRIBUTE_UNUSED) {
275  // No linker patches by default.
276}
277
278void CodeGenerator::InitializeCodeGeneration(size_t number_of_spill_slots,
279                                             size_t maximum_number_of_live_core_registers,
280                                             size_t maximum_number_of_live_fpu_registers,
281                                             size_t number_of_out_slots,
282                                             const ArenaVector<HBasicBlock*>& block_order) {
283  block_order_ = &block_order;
284  DCHECK(!block_order.empty());
285  DCHECK(block_order[0] == GetGraph()->GetEntryBlock());
286  ComputeSpillMask();
287  first_register_slot_in_slow_path_ = (number_of_out_slots + number_of_spill_slots) * kVRegSize;
288
289  if (number_of_spill_slots == 0
290      && !HasAllocatedCalleeSaveRegisters()
291      && IsLeafMethod()
292      && !RequiresCurrentMethod()) {
293    DCHECK_EQ(maximum_number_of_live_core_registers, 0u);
294    DCHECK_EQ(maximum_number_of_live_fpu_registers, 0u);
295    SetFrameSize(CallPushesPC() ? GetWordSize() : 0);
296  } else {
297    SetFrameSize(RoundUp(
298        number_of_spill_slots * kVRegSize
299        + number_of_out_slots * kVRegSize
300        + maximum_number_of_live_core_registers * GetWordSize()
301        + maximum_number_of_live_fpu_registers * GetFloatingPointSpillSlotSize()
302        + FrameEntrySpillSize(),
303        kStackAlignment));
304  }
305}
306
307void CodeGenerator::CreateCommonInvokeLocationSummary(
308    HInvoke* invoke, InvokeDexCallingConventionVisitor* visitor) {
309  ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena();
310  LocationSummary* locations = new (allocator) LocationSummary(invoke, LocationSummary::kCall);
311
312  for (size_t i = 0; i < invoke->GetNumberOfArguments(); i++) {
313    HInstruction* input = invoke->InputAt(i);
314    locations->SetInAt(i, visitor->GetNextLocation(input->GetType()));
315  }
316
317  locations->SetOut(visitor->GetReturnLocation(invoke->GetType()));
318
319  if (invoke->IsInvokeStaticOrDirect()) {
320    HInvokeStaticOrDirect* call = invoke->AsInvokeStaticOrDirect();
321    switch (call->GetMethodLoadKind()) {
322      case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
323        locations->SetInAt(call->GetSpecialInputIndex(), visitor->GetMethodLocation());
324        break;
325      case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod:
326        locations->AddTemp(visitor->GetMethodLocation());
327        locations->SetInAt(call->GetSpecialInputIndex(), Location::RequiresRegister());
328        break;
329      default:
330        locations->AddTemp(visitor->GetMethodLocation());
331        break;
332    }
333  } else {
334    locations->AddTemp(visitor->GetMethodLocation());
335  }
336}
337
338void CodeGenerator::GenerateInvokeUnresolvedRuntimeCall(HInvokeUnresolved* invoke) {
339  MoveConstant(invoke->GetLocations()->GetTemp(0), invoke->GetDexMethodIndex());
340
341  // Initialize to anything to silent compiler warnings.
342  QuickEntrypointEnum entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
343  switch (invoke->GetOriginalInvokeType()) {
344    case kStatic:
345      entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
346      break;
347    case kDirect:
348      entrypoint = kQuickInvokeDirectTrampolineWithAccessCheck;
349      break;
350    case kVirtual:
351      entrypoint = kQuickInvokeVirtualTrampolineWithAccessCheck;
352      break;
353    case kSuper:
354      entrypoint = kQuickInvokeSuperTrampolineWithAccessCheck;
355      break;
356    case kInterface:
357      entrypoint = kQuickInvokeInterfaceTrampolineWithAccessCheck;
358      break;
359  }
360  InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), nullptr);
361}
362
363void CodeGenerator::CreateUnresolvedFieldLocationSummary(
364    HInstruction* field_access,
365    Primitive::Type field_type,
366    const FieldAccessCallingConvention& calling_convention) {
367  bool is_instance = field_access->IsUnresolvedInstanceFieldGet()
368      || field_access->IsUnresolvedInstanceFieldSet();
369  bool is_get = field_access->IsUnresolvedInstanceFieldGet()
370      || field_access->IsUnresolvedStaticFieldGet();
371
372  ArenaAllocator* allocator = field_access->GetBlock()->GetGraph()->GetArena();
373  LocationSummary* locations =
374      new (allocator) LocationSummary(field_access, LocationSummary::kCall);
375
376  locations->AddTemp(calling_convention.GetFieldIndexLocation());
377
378  if (is_instance) {
379    // Add the `this` object for instance field accesses.
380    locations->SetInAt(0, calling_convention.GetObjectLocation());
381  }
382
383  // Note that pSetXXStatic/pGetXXStatic always takes/returns an int or int64
384  // regardless of the the type. Because of that we forced to special case
385  // the access to floating point values.
386  if (is_get) {
387    if (Primitive::IsFloatingPointType(field_type)) {
388      // The return value will be stored in regular registers while register
389      // allocator expects it in a floating point register.
390      // Note We don't need to request additional temps because the return
391      // register(s) are already blocked due the call and they may overlap with
392      // the input or field index.
393      // The transfer between the two will be done at codegen level.
394      locations->SetOut(calling_convention.GetFpuLocation(field_type));
395    } else {
396      locations->SetOut(calling_convention.GetReturnLocation(field_type));
397    }
398  } else {
399     size_t set_index = is_instance ? 1 : 0;
400     if (Primitive::IsFloatingPointType(field_type)) {
401      // The set value comes from a float location while the calling convention
402      // expects it in a regular register location. Allocate a temp for it and
403      // make the transfer at codegen.
404      AddLocationAsTemp(calling_convention.GetSetValueLocation(field_type, is_instance), locations);
405      locations->SetInAt(set_index, calling_convention.GetFpuLocation(field_type));
406    } else {
407      locations->SetInAt(set_index,
408          calling_convention.GetSetValueLocation(field_type, is_instance));
409    }
410  }
411}
412
413void CodeGenerator::GenerateUnresolvedFieldAccess(
414    HInstruction* field_access,
415    Primitive::Type field_type,
416    uint32_t field_index,
417    uint32_t dex_pc,
418    const FieldAccessCallingConvention& calling_convention) {
419  LocationSummary* locations = field_access->GetLocations();
420
421  MoveConstant(locations->GetTemp(0), field_index);
422
423  bool is_instance = field_access->IsUnresolvedInstanceFieldGet()
424      || field_access->IsUnresolvedInstanceFieldSet();
425  bool is_get = field_access->IsUnresolvedInstanceFieldGet()
426      || field_access->IsUnresolvedStaticFieldGet();
427
428  if (!is_get && Primitive::IsFloatingPointType(field_type)) {
429    // Copy the float value to be set into the calling convention register.
430    // Note that using directly the temp location is problematic as we don't
431    // support temp register pairs. To avoid boilerplate conversion code, use
432    // the location from the calling convention.
433    MoveLocation(calling_convention.GetSetValueLocation(field_type, is_instance),
434                 locations->InAt(is_instance ? 1 : 0),
435                 (Primitive::Is64BitType(field_type) ? Primitive::kPrimLong : Primitive::kPrimInt));
436  }
437
438  QuickEntrypointEnum entrypoint = kQuickSet8Static;  // Initialize to anything to avoid warnings.
439  switch (field_type) {
440    case Primitive::kPrimBoolean:
441      entrypoint = is_instance
442          ? (is_get ? kQuickGetBooleanInstance : kQuickSet8Instance)
443          : (is_get ? kQuickGetBooleanStatic : kQuickSet8Static);
444      break;
445    case Primitive::kPrimByte:
446      entrypoint = is_instance
447          ? (is_get ? kQuickGetByteInstance : kQuickSet8Instance)
448          : (is_get ? kQuickGetByteStatic : kQuickSet8Static);
449      break;
450    case Primitive::kPrimShort:
451      entrypoint = is_instance
452          ? (is_get ? kQuickGetShortInstance : kQuickSet16Instance)
453          : (is_get ? kQuickGetShortStatic : kQuickSet16Static);
454      break;
455    case Primitive::kPrimChar:
456      entrypoint = is_instance
457          ? (is_get ? kQuickGetCharInstance : kQuickSet16Instance)
458          : (is_get ? kQuickGetCharStatic : kQuickSet16Static);
459      break;
460    case Primitive::kPrimInt:
461    case Primitive::kPrimFloat:
462      entrypoint = is_instance
463          ? (is_get ? kQuickGet32Instance : kQuickSet32Instance)
464          : (is_get ? kQuickGet32Static : kQuickSet32Static);
465      break;
466    case Primitive::kPrimNot:
467      entrypoint = is_instance
468          ? (is_get ? kQuickGetObjInstance : kQuickSetObjInstance)
469          : (is_get ? kQuickGetObjStatic : kQuickSetObjStatic);
470      break;
471    case Primitive::kPrimLong:
472    case Primitive::kPrimDouble:
473      entrypoint = is_instance
474          ? (is_get ? kQuickGet64Instance : kQuickSet64Instance)
475          : (is_get ? kQuickGet64Static : kQuickSet64Static);
476      break;
477    default:
478      LOG(FATAL) << "Invalid type " << field_type;
479  }
480  InvokeRuntime(entrypoint, field_access, dex_pc, nullptr);
481
482  if (is_get && Primitive::IsFloatingPointType(field_type)) {
483    MoveLocation(locations->Out(), calling_convention.GetReturnLocation(field_type), field_type);
484  }
485}
486
487// TODO: Remove argument `code_generator_supports_read_barrier` when
488// all code generators have read barrier support.
489void CodeGenerator::CreateLoadClassLocationSummary(HLoadClass* cls,
490                                                   Location runtime_type_index_location,
491                                                   Location runtime_return_location,
492                                                   bool code_generator_supports_read_barrier) {
493  ArenaAllocator* allocator = cls->GetBlock()->GetGraph()->GetArena();
494  LocationSummary::CallKind call_kind = cls->NeedsAccessCheck()
495      ? LocationSummary::kCall
496      : (((code_generator_supports_read_barrier && kEmitCompilerReadBarrier) ||
497          cls->CanCallRuntime())
498            ? LocationSummary::kCallOnSlowPath
499            : LocationSummary::kNoCall);
500  LocationSummary* locations = new (allocator) LocationSummary(cls, call_kind);
501  if (cls->NeedsAccessCheck()) {
502    locations->SetInAt(0, Location::NoLocation());
503    locations->AddTemp(runtime_type_index_location);
504    locations->SetOut(runtime_return_location);
505  } else {
506    locations->SetInAt(0, Location::RequiresRegister());
507    locations->SetOut(Location::RequiresRegister());
508  }
509}
510
511
512void CodeGenerator::BlockIfInRegister(Location location, bool is_out) const {
513  // The DCHECKS below check that a register is not specified twice in
514  // the summary. The out location can overlap with an input, so we need
515  // to special case it.
516  if (location.IsRegister()) {
517    DCHECK(is_out || !blocked_core_registers_[location.reg()]);
518    blocked_core_registers_[location.reg()] = true;
519  } else if (location.IsFpuRegister()) {
520    DCHECK(is_out || !blocked_fpu_registers_[location.reg()]);
521    blocked_fpu_registers_[location.reg()] = true;
522  } else if (location.IsFpuRegisterPair()) {
523    DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()]);
524    blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()] = true;
525    DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()]);
526    blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()] = true;
527  } else if (location.IsRegisterPair()) {
528    DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairLow<int>()]);
529    blocked_core_registers_[location.AsRegisterPairLow<int>()] = true;
530    DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairHigh<int>()]);
531    blocked_core_registers_[location.AsRegisterPairHigh<int>()] = true;
532  }
533}
534
535void CodeGenerator::AllocateLocations(HInstruction* instruction) {
536  instruction->Accept(GetLocationBuilder());
537  DCHECK(CheckTypeConsistency(instruction));
538  LocationSummary* locations = instruction->GetLocations();
539  if (!instruction->IsSuspendCheckEntry()) {
540    if (locations != nullptr) {
541      if (locations->CanCall()) {
542        MarkNotLeaf();
543      } else if (locations->Intrinsified() &&
544                 instruction->IsInvokeStaticOrDirect() &&
545                 !instruction->AsInvokeStaticOrDirect()->HasCurrentMethodInput()) {
546        // A static method call that has been fully intrinsified, and cannot call on the slow
547        // path or refer to the current method directly, no longer needs current method.
548        return;
549      }
550    }
551    if (instruction->NeedsCurrentMethod()) {
552      SetRequiresCurrentMethod();
553    }
554  }
555}
556
557void CodeGenerator::MaybeRecordStat(MethodCompilationStat compilation_stat, size_t count) const {
558  if (stats_ != nullptr) {
559    stats_->RecordStat(compilation_stat, count);
560  }
561}
562
563std::unique_ptr<CodeGenerator> CodeGenerator::Create(HGraph* graph,
564                                                     InstructionSet instruction_set,
565                                                     const InstructionSetFeatures& isa_features,
566                                                     const CompilerOptions& compiler_options,
567                                                     OptimizingCompilerStats* stats) {
568  ArenaAllocator* arena = graph->GetArena();
569  switch (instruction_set) {
570#ifdef ART_ENABLE_CODEGEN_arm
571    case kArm:
572    case kThumb2: {
573      return std::unique_ptr<CodeGenerator>(
574          new (arena) arm::CodeGeneratorARM(graph,
575                                            *isa_features.AsArmInstructionSetFeatures(),
576                                            compiler_options,
577                                            stats));
578    }
579#endif
580#ifdef ART_ENABLE_CODEGEN_arm64
581    case kArm64: {
582      return std::unique_ptr<CodeGenerator>(
583          new (arena) arm64::CodeGeneratorARM64(graph,
584                                                *isa_features.AsArm64InstructionSetFeatures(),
585                                                compiler_options,
586                                                stats));
587    }
588#endif
589#ifdef ART_ENABLE_CODEGEN_mips
590    case kMips: {
591      return std::unique_ptr<CodeGenerator>(
592          new (arena) mips::CodeGeneratorMIPS(graph,
593                                              *isa_features.AsMipsInstructionSetFeatures(),
594                                              compiler_options,
595                                              stats));
596    }
597#endif
598#ifdef ART_ENABLE_CODEGEN_mips64
599    case kMips64: {
600      return std::unique_ptr<CodeGenerator>(
601          new (arena) mips64::CodeGeneratorMIPS64(graph,
602                                                  *isa_features.AsMips64InstructionSetFeatures(),
603                                                  compiler_options,
604                                                  stats));
605    }
606#endif
607#ifdef ART_ENABLE_CODEGEN_x86
608    case kX86: {
609      return std::unique_ptr<CodeGenerator>(
610          new (arena) x86::CodeGeneratorX86(graph,
611                                            *isa_features.AsX86InstructionSetFeatures(),
612                                            compiler_options,
613                                            stats));
614    }
615#endif
616#ifdef ART_ENABLE_CODEGEN_x86_64
617    case kX86_64: {
618      return std::unique_ptr<CodeGenerator>(
619          new (arena) x86_64::CodeGeneratorX86_64(graph,
620                                                  *isa_features.AsX86_64InstructionSetFeatures(),
621                                                  compiler_options,
622                                                  stats));
623    }
624#endif
625    default:
626      return nullptr;
627  }
628}
629
630size_t CodeGenerator::ComputeStackMapsSize() {
631  return stack_map_stream_.PrepareForFillIn();
632}
633
634static void CheckCovers(uint32_t dex_pc,
635                        const HGraph& graph,
636                        const CodeInfo& code_info,
637                        const ArenaVector<HSuspendCheck*>& loop_headers,
638                        ArenaVector<size_t>* covered) {
639  CodeInfoEncoding encoding = code_info.ExtractEncoding();
640  for (size_t i = 0; i < loop_headers.size(); ++i) {
641    if (loop_headers[i]->GetDexPc() == dex_pc) {
642      if (graph.IsCompilingOsr()) {
643        DCHECK(code_info.GetOsrStackMapForDexPc(dex_pc, encoding).IsValid());
644      }
645      ++(*covered)[i];
646    }
647  }
648}
649
650// Debug helper to ensure loop entries in compiled code are matched by
651// dex branch instructions.
652static void CheckLoopEntriesCanBeUsedForOsr(const HGraph& graph,
653                                            const CodeInfo& code_info,
654                                            const DexFile::CodeItem& code_item) {
655  if (graph.HasTryCatch()) {
656    // One can write loops through try/catch, which we do not support for OSR anyway.
657    return;
658  }
659  ArenaVector<HSuspendCheck*> loop_headers(graph.GetArena()->Adapter(kArenaAllocMisc));
660  for (HReversePostOrderIterator it(graph); !it.Done(); it.Advance()) {
661    if (it.Current()->IsLoopHeader()) {
662      HSuspendCheck* suspend_check = it.Current()->GetLoopInformation()->GetSuspendCheck();
663      if (!suspend_check->GetEnvironment()->IsFromInlinedInvoke()) {
664        loop_headers.push_back(suspend_check);
665      }
666    }
667  }
668  ArenaVector<size_t> covered(loop_headers.size(), 0, graph.GetArena()->Adapter(kArenaAllocMisc));
669  const uint16_t* code_ptr = code_item.insns_;
670  const uint16_t* code_end = code_item.insns_ + code_item.insns_size_in_code_units_;
671
672  size_t dex_pc = 0;
673  while (code_ptr < code_end) {
674    const Instruction& instruction = *Instruction::At(code_ptr);
675    if (instruction.IsBranch()) {
676      uint32_t target = dex_pc + instruction.GetTargetOffset();
677      CheckCovers(target, graph, code_info, loop_headers, &covered);
678    } else if (instruction.IsSwitch()) {
679      DexSwitchTable table(instruction, dex_pc);
680      uint16_t num_entries = table.GetNumEntries();
681      size_t offset = table.GetFirstValueIndex();
682
683      // Use a larger loop counter type to avoid overflow issues.
684      for (size_t i = 0; i < num_entries; ++i) {
685        // The target of the case.
686        uint32_t target = dex_pc + table.GetEntryAt(i + offset);
687        CheckCovers(target, graph, code_info, loop_headers, &covered);
688      }
689    }
690    dex_pc += instruction.SizeInCodeUnits();
691    code_ptr += instruction.SizeInCodeUnits();
692  }
693
694  for (size_t i = 0; i < covered.size(); ++i) {
695    DCHECK_NE(covered[i], 0u) << "Loop in compiled code has no dex branch equivalent";
696  }
697}
698
699void CodeGenerator::BuildStackMaps(MemoryRegion region, const DexFile::CodeItem& code_item) {
700  stack_map_stream_.FillIn(region);
701  if (kIsDebugBuild) {
702    CheckLoopEntriesCanBeUsedForOsr(*graph_, CodeInfo(region), code_item);
703  }
704}
705
706void CodeGenerator::RecordPcInfo(HInstruction* instruction,
707                                 uint32_t dex_pc,
708                                 SlowPathCode* slow_path) {
709  if (instruction != nullptr) {
710    // The code generated for some type conversions
711    // may call the runtime, thus normally requiring a subsequent
712    // call to this method. However, the method verifier does not
713    // produce PC information for certain instructions, which are
714    // considered "atomic" (they cannot join a GC).
715    // Therefore we do not currently record PC information for such
716    // instructions.  As this may change later, we added this special
717    // case so that code generators may nevertheless call
718    // CodeGenerator::RecordPcInfo without triggering an error in
719    // CodeGenerator::BuildNativeGCMap ("Missing ref for dex pc 0x")
720    // thereafter.
721    if (instruction->IsTypeConversion()) {
722      return;
723    }
724    if (instruction->IsRem()) {
725      Primitive::Type type = instruction->AsRem()->GetResultType();
726      if ((type == Primitive::kPrimFloat) || (type == Primitive::kPrimDouble)) {
727        return;
728      }
729    }
730  }
731
732  uint32_t outer_dex_pc = dex_pc;
733  uint32_t outer_environment_size = 0;
734  uint32_t inlining_depth = 0;
735  if (instruction != nullptr) {
736    for (HEnvironment* environment = instruction->GetEnvironment();
737         environment != nullptr;
738         environment = environment->GetParent()) {
739      outer_dex_pc = environment->GetDexPc();
740      outer_environment_size = environment->Size();
741      if (environment != instruction->GetEnvironment()) {
742        inlining_depth++;
743      }
744    }
745  }
746
747  // Collect PC infos for the mapping table.
748  uint32_t native_pc = GetAssembler()->CodeSize();
749
750  if (instruction == nullptr) {
751    // For stack overflow checks and native-debug-info entries without dex register
752    // mapping (i.e. start of basic block or start of slow path).
753    stack_map_stream_.BeginStackMapEntry(outer_dex_pc, native_pc, 0, 0, 0, 0);
754    stack_map_stream_.EndStackMapEntry();
755    return;
756  }
757  LocationSummary* locations = instruction->GetLocations();
758
759  uint32_t register_mask = locations->GetRegisterMask();
760  if (locations->OnlyCallsOnSlowPath()) {
761    // In case of slow path, we currently set the location of caller-save registers
762    // to register (instead of their stack location when pushed before the slow-path
763    // call). Therefore register_mask contains both callee-save and caller-save
764    // registers that hold objects. We must remove the caller-save from the mask, since
765    // they will be overwritten by the callee.
766    register_mask &= core_callee_save_mask_;
767  }
768  // The register mask must be a subset of callee-save registers.
769  DCHECK_EQ(register_mask & core_callee_save_mask_, register_mask);
770  stack_map_stream_.BeginStackMapEntry(outer_dex_pc,
771                                       native_pc,
772                                       register_mask,
773                                       locations->GetStackMask(),
774                                       outer_environment_size,
775                                       inlining_depth);
776
777  EmitEnvironment(instruction->GetEnvironment(), slow_path);
778  stack_map_stream_.EndStackMapEntry();
779
780  HLoopInformation* info = instruction->GetBlock()->GetLoopInformation();
781  if (instruction->IsSuspendCheck() &&
782      (info != nullptr) &&
783      graph_->IsCompilingOsr() &&
784      (inlining_depth == 0)) {
785    DCHECK_EQ(info->GetSuspendCheck(), instruction);
786    // We duplicate the stack map as a marker that this stack map can be an OSR entry.
787    // Duplicating it avoids having the runtime recognize and skip an OSR stack map.
788    DCHECK(info->IsIrreducible());
789    stack_map_stream_.BeginStackMapEntry(
790        dex_pc, native_pc, register_mask, locations->GetStackMask(), outer_environment_size, 0);
791    EmitEnvironment(instruction->GetEnvironment(), slow_path);
792    stack_map_stream_.EndStackMapEntry();
793    if (kIsDebugBuild) {
794      HEnvironment* environment = instruction->GetEnvironment();
795      for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) {
796        HInstruction* in_environment = environment->GetInstructionAt(i);
797        if (in_environment != nullptr) {
798          DCHECK(in_environment->IsPhi() || in_environment->IsConstant());
799          Location location = environment->GetLocationAt(i);
800          DCHECK(location.IsStackSlot() ||
801                 location.IsDoubleStackSlot() ||
802                 location.IsConstant() ||
803                 location.IsInvalid());
804          if (location.IsStackSlot() || location.IsDoubleStackSlot()) {
805            DCHECK_LT(location.GetStackIndex(), static_cast<int32_t>(GetFrameSize()));
806          }
807        }
808      }
809    }
810  } else if (kIsDebugBuild) {
811    // Ensure stack maps are unique, by checking that the native pc in the stack map
812    // last emitted is different than the native pc of the stack map just emitted.
813    size_t number_of_stack_maps = stack_map_stream_.GetNumberOfStackMaps();
814    if (number_of_stack_maps > 1) {
815      DCHECK_NE(stack_map_stream_.GetStackMap(number_of_stack_maps - 1).native_pc_offset,
816                stack_map_stream_.GetStackMap(number_of_stack_maps - 2).native_pc_offset);
817    }
818  }
819}
820
821bool CodeGenerator::HasStackMapAtCurrentPc() {
822  uint32_t pc = GetAssembler()->CodeSize();
823  size_t count = stack_map_stream_.GetNumberOfStackMaps();
824  return count > 0 && stack_map_stream_.GetStackMap(count - 1).native_pc_offset == pc;
825}
826
827void CodeGenerator::MaybeRecordNativeDebugInfo(HInstruction* instruction,
828                                               uint32_t dex_pc,
829                                               SlowPathCode* slow_path) {
830  if (GetCompilerOptions().GetNativeDebuggable() && dex_pc != kNoDexPc) {
831    if (HasStackMapAtCurrentPc()) {
832      // Ensure that we do not collide with the stack map of the previous instruction.
833      GenerateNop();
834    }
835    RecordPcInfo(instruction, dex_pc, slow_path);
836  }
837}
838
839void CodeGenerator::RecordCatchBlockInfo() {
840  ArenaAllocator* arena = graph_->GetArena();
841
842  for (HBasicBlock* block : *block_order_) {
843    if (!block->IsCatchBlock()) {
844      continue;
845    }
846
847    uint32_t dex_pc = block->GetDexPc();
848    uint32_t num_vregs = graph_->GetNumberOfVRegs();
849    uint32_t inlining_depth = 0;  // Inlining of catch blocks is not supported at the moment.
850    uint32_t native_pc = GetAddressOf(block);
851    uint32_t register_mask = 0;   // Not used.
852
853    // The stack mask is not used, so we leave it empty.
854    ArenaBitVector* stack_mask =
855        ArenaBitVector::Create(arena, 0, /* expandable */ true, kArenaAllocCodeGenerator);
856
857    stack_map_stream_.BeginStackMapEntry(dex_pc,
858                                         native_pc,
859                                         register_mask,
860                                         stack_mask,
861                                         num_vregs,
862                                         inlining_depth);
863
864    HInstruction* current_phi = block->GetFirstPhi();
865    for (size_t vreg = 0; vreg < num_vregs; ++vreg) {
866    while (current_phi != nullptr && current_phi->AsPhi()->GetRegNumber() < vreg) {
867      HInstruction* next_phi = current_phi->GetNext();
868      DCHECK(next_phi == nullptr ||
869             current_phi->AsPhi()->GetRegNumber() <= next_phi->AsPhi()->GetRegNumber())
870          << "Phis need to be sorted by vreg number to keep this a linear-time loop.";
871      current_phi = next_phi;
872    }
873
874      if (current_phi == nullptr || current_phi->AsPhi()->GetRegNumber() != vreg) {
875        stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
876      } else {
877        Location location = current_phi->GetLiveInterval()->ToLocation();
878        switch (location.GetKind()) {
879          case Location::kStackSlot: {
880            stack_map_stream_.AddDexRegisterEntry(
881                DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
882            break;
883          }
884          case Location::kDoubleStackSlot: {
885            stack_map_stream_.AddDexRegisterEntry(
886                DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
887            stack_map_stream_.AddDexRegisterEntry(
888                DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize));
889            ++vreg;
890            DCHECK_LT(vreg, num_vregs);
891            break;
892          }
893          default: {
894            // All catch phis must be allocated to a stack slot.
895            LOG(FATAL) << "Unexpected kind " << location.GetKind();
896            UNREACHABLE();
897          }
898        }
899      }
900    }
901
902    stack_map_stream_.EndStackMapEntry();
903  }
904}
905
906void CodeGenerator::EmitEnvironment(HEnvironment* environment, SlowPathCode* slow_path) {
907  if (environment == nullptr) return;
908
909  if (environment->GetParent() != nullptr) {
910    // We emit the parent environment first.
911    EmitEnvironment(environment->GetParent(), slow_path);
912    stack_map_stream_.BeginInlineInfoEntry(environment->GetMethodIdx(),
913                                           environment->GetDexPc(),
914                                           environment->GetInvokeType(),
915                                           environment->Size());
916  }
917
918  // Walk over the environment, and record the location of dex registers.
919  for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) {
920    HInstruction* current = environment->GetInstructionAt(i);
921    if (current == nullptr) {
922      stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
923      continue;
924    }
925
926    Location location = environment->GetLocationAt(i);
927    switch (location.GetKind()) {
928      case Location::kConstant: {
929        DCHECK_EQ(current, location.GetConstant());
930        if (current->IsLongConstant()) {
931          int64_t value = current->AsLongConstant()->GetValue();
932          stack_map_stream_.AddDexRegisterEntry(
933              DexRegisterLocation::Kind::kConstant, Low32Bits(value));
934          stack_map_stream_.AddDexRegisterEntry(
935              DexRegisterLocation::Kind::kConstant, High32Bits(value));
936          ++i;
937          DCHECK_LT(i, environment_size);
938        } else if (current->IsDoubleConstant()) {
939          int64_t value = bit_cast<int64_t, double>(current->AsDoubleConstant()->GetValue());
940          stack_map_stream_.AddDexRegisterEntry(
941              DexRegisterLocation::Kind::kConstant, Low32Bits(value));
942          stack_map_stream_.AddDexRegisterEntry(
943              DexRegisterLocation::Kind::kConstant, High32Bits(value));
944          ++i;
945          DCHECK_LT(i, environment_size);
946        } else if (current->IsIntConstant()) {
947          int32_t value = current->AsIntConstant()->GetValue();
948          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, value);
949        } else if (current->IsNullConstant()) {
950          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, 0);
951        } else {
952          DCHECK(current->IsFloatConstant()) << current->DebugName();
953          int32_t value = bit_cast<int32_t, float>(current->AsFloatConstant()->GetValue());
954          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, value);
955        }
956        break;
957      }
958
959      case Location::kStackSlot: {
960        stack_map_stream_.AddDexRegisterEntry(
961            DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
962        break;
963      }
964
965      case Location::kDoubleStackSlot: {
966        stack_map_stream_.AddDexRegisterEntry(
967            DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
968        stack_map_stream_.AddDexRegisterEntry(
969            DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize));
970        ++i;
971        DCHECK_LT(i, environment_size);
972        break;
973      }
974
975      case Location::kRegister : {
976        int id = location.reg();
977        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(id)) {
978          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(id);
979          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
980          if (current->GetType() == Primitive::kPrimLong) {
981            stack_map_stream_.AddDexRegisterEntry(
982                DexRegisterLocation::Kind::kInStack, offset + kVRegSize);
983            ++i;
984            DCHECK_LT(i, environment_size);
985          }
986        } else {
987          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, id);
988          if (current->GetType() == Primitive::kPrimLong) {
989            stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegisterHigh, id);
990            ++i;
991            DCHECK_LT(i, environment_size);
992          }
993        }
994        break;
995      }
996
997      case Location::kFpuRegister : {
998        int id = location.reg();
999        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(id)) {
1000          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(id);
1001          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1002          if (current->GetType() == Primitive::kPrimDouble) {
1003            stack_map_stream_.AddDexRegisterEntry(
1004                DexRegisterLocation::Kind::kInStack, offset + kVRegSize);
1005            ++i;
1006            DCHECK_LT(i, environment_size);
1007          }
1008        } else {
1009          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, id);
1010          if (current->GetType() == Primitive::kPrimDouble) {
1011            stack_map_stream_.AddDexRegisterEntry(
1012                DexRegisterLocation::Kind::kInFpuRegisterHigh, id);
1013            ++i;
1014            DCHECK_LT(i, environment_size);
1015          }
1016        }
1017        break;
1018      }
1019
1020      case Location::kFpuRegisterPair : {
1021        int low = location.low();
1022        int high = location.high();
1023        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(low)) {
1024          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(low);
1025          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1026        } else {
1027          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, low);
1028        }
1029        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(high)) {
1030          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(high);
1031          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1032          ++i;
1033        } else {
1034          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, high);
1035          ++i;
1036        }
1037        DCHECK_LT(i, environment_size);
1038        break;
1039      }
1040
1041      case Location::kRegisterPair : {
1042        int low = location.low();
1043        int high = location.high();
1044        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(low)) {
1045          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(low);
1046          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1047        } else {
1048          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, low);
1049        }
1050        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(high)) {
1051          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(high);
1052          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1053        } else {
1054          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, high);
1055        }
1056        ++i;
1057        DCHECK_LT(i, environment_size);
1058        break;
1059      }
1060
1061      case Location::kInvalid: {
1062        stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
1063        break;
1064      }
1065
1066      default:
1067        LOG(FATAL) << "Unexpected kind " << location.GetKind();
1068    }
1069  }
1070
1071  if (environment->GetParent() != nullptr) {
1072    stack_map_stream_.EndInlineInfoEntry();
1073  }
1074}
1075
1076bool CodeGenerator::IsImplicitNullCheckAllowed(HNullCheck* null_check) const {
1077  return compiler_options_.GetImplicitNullChecks() &&
1078         // Null checks which might throw into a catch block need to save live
1079         // registers and therefore cannot be done implicitly.
1080         !null_check->CanThrowIntoCatchBlock();
1081}
1082
1083bool CodeGenerator::CanMoveNullCheckToUser(HNullCheck* null_check) {
1084  HInstruction* first_next_not_move = null_check->GetNextDisregardingMoves();
1085
1086  return (first_next_not_move != nullptr)
1087      && first_next_not_move->CanDoImplicitNullCheckOn(null_check->InputAt(0));
1088}
1089
1090void CodeGenerator::MaybeRecordImplicitNullCheck(HInstruction* instr) {
1091  // If we are from a static path don't record the pc as we can't throw NPE.
1092  // NB: having the checks here makes the code much less verbose in the arch
1093  // specific code generators.
1094  if (instr->IsStaticFieldSet() || instr->IsStaticFieldGet()) {
1095    return;
1096  }
1097
1098  if (!instr->CanDoImplicitNullCheckOn(instr->InputAt(0))) {
1099    return;
1100  }
1101
1102  // Find the first previous instruction which is not a move.
1103  HInstruction* first_prev_not_move = instr->GetPreviousDisregardingMoves();
1104
1105  // If the instruction is a null check it means that `instr` is the first user
1106  // and needs to record the pc.
1107  if (first_prev_not_move != nullptr && first_prev_not_move->IsNullCheck()) {
1108    HNullCheck* null_check = first_prev_not_move->AsNullCheck();
1109    if (IsImplicitNullCheckAllowed(null_check)) {
1110      // TODO: The parallel moves modify the environment. Their changes need to be
1111      // reverted otherwise the stack maps at the throw point will not be correct.
1112      RecordPcInfo(null_check, null_check->GetDexPc());
1113    }
1114  }
1115}
1116
1117void CodeGenerator::GenerateNullCheck(HNullCheck* instruction) {
1118  if (IsImplicitNullCheckAllowed(instruction)) {
1119    MaybeRecordStat(kImplicitNullCheckGenerated);
1120    GenerateImplicitNullCheck(instruction);
1121  } else {
1122    MaybeRecordStat(kExplicitNullCheckGenerated);
1123    GenerateExplicitNullCheck(instruction);
1124  }
1125}
1126
1127void CodeGenerator::ClearSpillSlotsFromLoopPhisInStackMap(HSuspendCheck* suspend_check) const {
1128  LocationSummary* locations = suspend_check->GetLocations();
1129  HBasicBlock* block = suspend_check->GetBlock();
1130  DCHECK(block->GetLoopInformation()->GetSuspendCheck() == suspend_check);
1131  DCHECK(block->IsLoopHeader());
1132
1133  for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) {
1134    HInstruction* current = it.Current();
1135    LiveInterval* interval = current->GetLiveInterval();
1136    // We only need to clear bits of loop phis containing objects and allocated in register.
1137    // Loop phis allocated on stack already have the object in the stack.
1138    if (current->GetType() == Primitive::kPrimNot
1139        && interval->HasRegister()
1140        && interval->HasSpillSlot()) {
1141      locations->ClearStackBit(interval->GetSpillSlot() / kVRegSize);
1142    }
1143  }
1144}
1145
1146void CodeGenerator::EmitParallelMoves(Location from1,
1147                                      Location to1,
1148                                      Primitive::Type type1,
1149                                      Location from2,
1150                                      Location to2,
1151                                      Primitive::Type type2) {
1152  HParallelMove parallel_move(GetGraph()->GetArena());
1153  parallel_move.AddMove(from1, to1, type1, nullptr);
1154  parallel_move.AddMove(from2, to2, type2, nullptr);
1155  GetMoveResolver()->EmitNativeCode(&parallel_move);
1156}
1157
1158void CodeGenerator::ValidateInvokeRuntime(HInstruction* instruction, SlowPathCode* slow_path) {
1159  // Ensure that the call kind indication given to the register allocator is
1160  // coherent with the runtime call generated, and that the GC side effect is
1161  // set when required.
1162  if (slow_path == nullptr) {
1163    DCHECK(instruction->GetLocations()->WillCall())
1164        << "instruction->DebugName()=" << instruction->DebugName();
1165    DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()))
1166        << "instruction->DebugName()=" << instruction->DebugName()
1167        << " instruction->GetSideEffects().ToString()=" << instruction->GetSideEffects().ToString();
1168  } else {
1169    DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath() || slow_path->IsFatal())
1170        << "instruction->DebugName()=" << instruction->DebugName()
1171        << " slow_path->GetDescription()=" << slow_path->GetDescription();
1172    DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()) ||
1173           // When read barriers are enabled, some instructions use a
1174           // slow path to emit a read barrier, which does not trigger
1175           // GC, is not fatal, nor is emitted by HDeoptimize
1176           // instructions.
1177           (kEmitCompilerReadBarrier &&
1178            (instruction->IsInstanceFieldGet() ||
1179             instruction->IsStaticFieldGet() ||
1180             instruction->IsArraySet() ||
1181             instruction->IsArrayGet() ||
1182             instruction->IsLoadClass() ||
1183             instruction->IsLoadString() ||
1184             instruction->IsInstanceOf() ||
1185             instruction->IsCheckCast())))
1186        << "instruction->DebugName()=" << instruction->DebugName()
1187        << " instruction->GetSideEffects().ToString()=" << instruction->GetSideEffects().ToString()
1188        << " slow_path->GetDescription()=" << slow_path->GetDescription();
1189  }
1190
1191  // Check the coherency of leaf information.
1192  DCHECK(instruction->IsSuspendCheck()
1193         || ((slow_path != nullptr) && slow_path->IsFatal())
1194         || instruction->GetLocations()->CanCall()
1195         || !IsLeafMethod())
1196      << instruction->DebugName() << ((slow_path != nullptr) ? slow_path->GetDescription() : "");
1197}
1198
1199void SlowPathCode::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
1200  RegisterSet* live_registers = locations->GetLiveRegisters();
1201  size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
1202
1203  for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
1204    if (!codegen->IsCoreCalleeSaveRegister(i)) {
1205      if (live_registers->ContainsCoreRegister(i)) {
1206        // If the register holds an object, update the stack mask.
1207        if (locations->RegisterContainsObject(i)) {
1208          locations->SetStackBit(stack_offset / kVRegSize);
1209        }
1210        DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1211        DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1212        saved_core_stack_offsets_[i] = stack_offset;
1213        stack_offset += codegen->SaveCoreRegister(stack_offset, i);
1214      }
1215    }
1216  }
1217
1218  for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) {
1219    if (!codegen->IsFloatingPointCalleeSaveRegister(i)) {
1220      if (live_registers->ContainsFloatingPointRegister(i)) {
1221        DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1222        DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1223        saved_fpu_stack_offsets_[i] = stack_offset;
1224        stack_offset += codegen->SaveFloatingPointRegister(stack_offset, i);
1225      }
1226    }
1227  }
1228}
1229
1230void SlowPathCode::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
1231  RegisterSet* live_registers = locations->GetLiveRegisters();
1232  size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
1233
1234  for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
1235    if (!codegen->IsCoreCalleeSaveRegister(i)) {
1236      if (live_registers->ContainsCoreRegister(i)) {
1237        DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1238        DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1239        stack_offset += codegen->RestoreCoreRegister(stack_offset, i);
1240      }
1241    }
1242  }
1243
1244  for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) {
1245    if (!codegen->IsFloatingPointCalleeSaveRegister(i)) {
1246      if (live_registers->ContainsFloatingPointRegister(i)) {
1247        DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1248        DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1249        stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, i);
1250      }
1251    }
1252  }
1253}
1254
1255void CodeGenerator::CreateSystemArrayCopyLocationSummary(HInvoke* invoke) {
1256  // Check to see if we have known failures that will cause us to have to bail out
1257  // to the runtime, and just generate the runtime call directly.
1258  HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant();
1259  HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant();
1260
1261  // The positions must be non-negative.
1262  if ((src_pos != nullptr && src_pos->GetValue() < 0) ||
1263      (dest_pos != nullptr && dest_pos->GetValue() < 0)) {
1264    // We will have to fail anyways.
1265    return;
1266  }
1267
1268  // The length must be >= 0.
1269  HIntConstant* length = invoke->InputAt(4)->AsIntConstant();
1270  if (length != nullptr) {
1271    int32_t len = length->GetValue();
1272    if (len < 0) {
1273      // Just call as normal.
1274      return;
1275    }
1276  }
1277
1278  SystemArrayCopyOptimizations optimizations(invoke);
1279
1280  if (optimizations.GetDestinationIsSource()) {
1281    if (src_pos != nullptr && dest_pos != nullptr && src_pos->GetValue() < dest_pos->GetValue()) {
1282      // We only support backward copying if source and destination are the same.
1283      return;
1284    }
1285  }
1286
1287  if (optimizations.GetDestinationIsPrimitiveArray() || optimizations.GetSourceIsPrimitiveArray()) {
1288    // We currently don't intrinsify primitive copying.
1289    return;
1290  }
1291
1292  ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena();
1293  LocationSummary* locations = new (allocator) LocationSummary(invoke,
1294                                                               LocationSummary::kCallOnSlowPath,
1295                                                               kIntrinsified);
1296  // arraycopy(Object src, int src_pos, Object dest, int dest_pos, int length).
1297  locations->SetInAt(0, Location::RequiresRegister());
1298  locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1)));
1299  locations->SetInAt(2, Location::RequiresRegister());
1300  locations->SetInAt(3, Location::RegisterOrConstant(invoke->InputAt(3)));
1301  locations->SetInAt(4, Location::RegisterOrConstant(invoke->InputAt(4)));
1302
1303  locations->AddTemp(Location::RequiresRegister());
1304  locations->AddTemp(Location::RequiresRegister());
1305  locations->AddTemp(Location::RequiresRegister());
1306}
1307
1308}  // namespace art
1309