code_generator.cc revision e460d1df1f789c7c8bb97024a8efbd713ac175e9 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator.h"
18
19#ifdef ART_ENABLE_CODEGEN_arm
20#include "code_generator_arm.h"
21#endif
22
23#ifdef ART_ENABLE_CODEGEN_arm64
24#include "code_generator_arm64.h"
25#endif
26
27#ifdef ART_ENABLE_CODEGEN_x86
28#include "code_generator_x86.h"
29#endif
30
31#ifdef ART_ENABLE_CODEGEN_x86_64
32#include "code_generator_x86_64.h"
33#endif
34
35#ifdef ART_ENABLE_CODEGEN_mips64
36#include "code_generator_mips64.h"
37#endif
38
39#include "compiled_method.h"
40#include "dex/verified_method.h"
41#include "driver/dex_compilation_unit.h"
42#include "gc_map_builder.h"
43#include "graph_visualizer.h"
44#include "leb128.h"
45#include "mapping_table.h"
46#include "mirror/array-inl.h"
47#include "mirror/object_array-inl.h"
48#include "mirror/object_reference.h"
49#include "parallel_move_resolver.h"
50#include "ssa_liveness_analysis.h"
51#include "utils/assembler.h"
52#include "verifier/dex_gc_map.h"
53#include "vmap_table.h"
54
55namespace art {
56
57// Return whether a location is consistent with a type.
58static bool CheckType(Primitive::Type type, Location location) {
59  if (location.IsFpuRegister()
60      || (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresFpuRegister))) {
61    return (type == Primitive::kPrimFloat) || (type == Primitive::kPrimDouble);
62  } else if (location.IsRegister() ||
63             (location.IsUnallocated() && (location.GetPolicy() == Location::kRequiresRegister))) {
64    return Primitive::IsIntegralType(type) || (type == Primitive::kPrimNot);
65  } else if (location.IsRegisterPair()) {
66    return type == Primitive::kPrimLong;
67  } else if (location.IsFpuRegisterPair()) {
68    return type == Primitive::kPrimDouble;
69  } else if (location.IsStackSlot()) {
70    return (Primitive::IsIntegralType(type) && type != Primitive::kPrimLong)
71           || (type == Primitive::kPrimFloat)
72           || (type == Primitive::kPrimNot);
73  } else if (location.IsDoubleStackSlot()) {
74    return (type == Primitive::kPrimLong) || (type == Primitive::kPrimDouble);
75  } else if (location.IsConstant()) {
76    if (location.GetConstant()->IsIntConstant()) {
77      return Primitive::IsIntegralType(type) && (type != Primitive::kPrimLong);
78    } else if (location.GetConstant()->IsNullConstant()) {
79      return type == Primitive::kPrimNot;
80    } else if (location.GetConstant()->IsLongConstant()) {
81      return type == Primitive::kPrimLong;
82    } else if (location.GetConstant()->IsFloatConstant()) {
83      return type == Primitive::kPrimFloat;
84    } else {
85      return location.GetConstant()->IsDoubleConstant()
86          && (type == Primitive::kPrimDouble);
87    }
88  } else {
89    return location.IsInvalid() || (location.GetPolicy() == Location::kAny);
90  }
91}
92
93// Check that a location summary is consistent with an instruction.
94static bool CheckTypeConsistency(HInstruction* instruction) {
95  LocationSummary* locations = instruction->GetLocations();
96  if (locations == nullptr) {
97    return true;
98  }
99
100  if (locations->Out().IsUnallocated()
101      && (locations->Out().GetPolicy() == Location::kSameAsFirstInput)) {
102    DCHECK(CheckType(instruction->GetType(), locations->InAt(0)))
103        << instruction->GetType()
104        << " " << locations->InAt(0);
105  } else {
106    DCHECK(CheckType(instruction->GetType(), locations->Out()))
107        << instruction->GetType()
108        << " " << locations->Out();
109  }
110
111  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
112    DCHECK(CheckType(instruction->InputAt(i)->GetType(), locations->InAt(i)))
113      << instruction->InputAt(i)->GetType()
114      << " " << locations->InAt(i);
115  }
116
117  HEnvironment* environment = instruction->GetEnvironment();
118  for (size_t i = 0; i < instruction->EnvironmentSize(); ++i) {
119    if (environment->GetInstructionAt(i) != nullptr) {
120      Primitive::Type type = environment->GetInstructionAt(i)->GetType();
121      DCHECK(CheckType(type, environment->GetLocationAt(i)))
122        << type << " " << environment->GetLocationAt(i);
123    } else {
124      DCHECK(environment->GetLocationAt(i).IsInvalid())
125        << environment->GetLocationAt(i);
126    }
127  }
128  return true;
129}
130
131size_t CodeGenerator::GetCacheOffset(uint32_t index) {
132  return sizeof(GcRoot<mirror::Object>) * index;
133}
134
135size_t CodeGenerator::GetCachePointerOffset(uint32_t index) {
136  auto pointer_size = InstructionSetPointerSize(GetInstructionSet());
137  return pointer_size * index;
138}
139
140void CodeGenerator::CompileBaseline(CodeAllocator* allocator, bool is_leaf) {
141  Initialize();
142  if (!is_leaf) {
143    MarkNotLeaf();
144  }
145  const bool is_64_bit = Is64BitInstructionSet(GetInstructionSet());
146  InitializeCodeGeneration(GetGraph()->GetNumberOfLocalVRegs()
147                             + GetGraph()->GetTemporariesVRegSlots()
148                             + 1 /* filler */,
149                           0, /* the baseline compiler does not have live registers at slow path */
150                           0, /* the baseline compiler does not have live registers at slow path */
151                           GetGraph()->GetMaximumNumberOfOutVRegs()
152                             + (is_64_bit ? 2 : 1) /* current method */,
153                           GetGraph()->GetBlocks());
154  CompileInternal(allocator, /* is_baseline */ true);
155}
156
157bool CodeGenerator::GoesToNextBlock(HBasicBlock* current, HBasicBlock* next) const {
158  DCHECK_LT(current_block_index_, block_order_->size());
159  DCHECK_EQ((*block_order_)[current_block_index_], current);
160  return GetNextBlockToEmit() == FirstNonEmptyBlock(next);
161}
162
163HBasicBlock* CodeGenerator::GetNextBlockToEmit() const {
164  for (size_t i = current_block_index_ + 1; i < block_order_->size(); ++i) {
165    HBasicBlock* block = (*block_order_)[i];
166    if (!block->IsSingleJump()) {
167      return block;
168    }
169  }
170  return nullptr;
171}
172
173HBasicBlock* CodeGenerator::FirstNonEmptyBlock(HBasicBlock* block) const {
174  while (block->IsSingleJump()) {
175    block = block->GetSuccessor(0);
176  }
177  return block;
178}
179
180class DisassemblyScope {
181 public:
182  DisassemblyScope(HInstruction* instruction, const CodeGenerator& codegen)
183      : codegen_(codegen), instruction_(instruction), start_offset_(static_cast<size_t>(-1)) {
184    if (codegen_.GetDisassemblyInformation() != nullptr) {
185      start_offset_ = codegen_.GetAssembler().CodeSize();
186    }
187  }
188
189  ~DisassemblyScope() {
190    // We avoid building this data when we know it will not be used.
191    if (codegen_.GetDisassemblyInformation() != nullptr) {
192      codegen_.GetDisassemblyInformation()->AddInstructionInterval(
193          instruction_, start_offset_, codegen_.GetAssembler().CodeSize());
194    }
195  }
196
197 private:
198  const CodeGenerator& codegen_;
199  HInstruction* instruction_;
200  size_t start_offset_;
201};
202
203
204void CodeGenerator::GenerateSlowPaths() {
205  size_t code_start = 0;
206  for (SlowPathCode* slow_path : slow_paths_) {
207    if (disasm_info_ != nullptr) {
208      code_start = GetAssembler()->CodeSize();
209    }
210    slow_path->EmitNativeCode(this);
211    if (disasm_info_ != nullptr) {
212      disasm_info_->AddSlowPathInterval(slow_path, code_start, GetAssembler()->CodeSize());
213    }
214  }
215}
216
217void CodeGenerator::CompileInternal(CodeAllocator* allocator, bool is_baseline) {
218  is_baseline_ = is_baseline;
219  HGraphVisitor* instruction_visitor = GetInstructionVisitor();
220  DCHECK_EQ(current_block_index_, 0u);
221
222  size_t frame_start = GetAssembler()->CodeSize();
223  GenerateFrameEntry();
224  DCHECK_EQ(GetAssembler()->cfi().GetCurrentCFAOffset(), static_cast<int>(frame_size_));
225  if (disasm_info_ != nullptr) {
226    disasm_info_->SetFrameEntryInterval(frame_start, GetAssembler()->CodeSize());
227  }
228
229  for (size_t e = block_order_->size(); current_block_index_ < e; ++current_block_index_) {
230    HBasicBlock* block = (*block_order_)[current_block_index_];
231    // Don't generate code for an empty block. Its predecessors will branch to its successor
232    // directly. Also, the label of that block will not be emitted, so this helps catch
233    // errors where we reference that label.
234    if (block->IsSingleJump()) continue;
235    Bind(block);
236    for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) {
237      HInstruction* current = it.Current();
238      DisassemblyScope disassembly_scope(current, *this);
239      if (is_baseline) {
240        InitLocationsBaseline(current);
241      }
242      DCHECK(CheckTypeConsistency(current));
243      uintptr_t native_pc_begin = GetAssembler()->CodeSize();
244      current->Accept(instruction_visitor);
245      uintptr_t native_pc_end = GetAssembler()->CodeSize();
246      RecordNativeDebugInfo(current->GetDexPc(), native_pc_begin, native_pc_end);
247    }
248  }
249
250  GenerateSlowPaths();
251
252  // Emit catch stack maps at the end of the stack map stream as expected by the
253  // runtime exception handler.
254  if (!is_baseline && graph_->HasTryCatch()) {
255    RecordCatchBlockInfo();
256  }
257
258  // Finalize instructions in assember;
259  Finalize(allocator);
260}
261
262void CodeGenerator::CompileOptimized(CodeAllocator* allocator) {
263  // The register allocator already called `InitializeCodeGeneration`,
264  // where the frame size has been computed.
265  DCHECK(block_order_ != nullptr);
266  Initialize();
267  CompileInternal(allocator, /* is_baseline */ false);
268}
269
270void CodeGenerator::Finalize(CodeAllocator* allocator) {
271  size_t code_size = GetAssembler()->CodeSize();
272  uint8_t* buffer = allocator->Allocate(code_size);
273
274  MemoryRegion code(buffer, code_size);
275  GetAssembler()->FinalizeInstructions(code);
276}
277
278void CodeGenerator::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches ATTRIBUTE_UNUSED) {
279  // No linker patches by default.
280}
281
282size_t CodeGenerator::FindFreeEntry(bool* array, size_t length) {
283  for (size_t i = 0; i < length; ++i) {
284    if (!array[i]) {
285      array[i] = true;
286      return i;
287    }
288  }
289  LOG(FATAL) << "Could not find a register in baseline register allocator";
290  UNREACHABLE();
291}
292
293size_t CodeGenerator::FindTwoFreeConsecutiveAlignedEntries(bool* array, size_t length) {
294  for (size_t i = 0; i < length - 1; i += 2) {
295    if (!array[i] && !array[i + 1]) {
296      array[i] = true;
297      array[i + 1] = true;
298      return i;
299    }
300  }
301  LOG(FATAL) << "Could not find a register in baseline register allocator";
302  UNREACHABLE();
303}
304
305void CodeGenerator::InitializeCodeGeneration(size_t number_of_spill_slots,
306                                             size_t maximum_number_of_live_core_registers,
307                                             size_t maximum_number_of_live_fp_registers,
308                                             size_t number_of_out_slots,
309                                             const ArenaVector<HBasicBlock*>& block_order) {
310  block_order_ = &block_order;
311  DCHECK(!block_order.empty());
312  DCHECK(block_order[0] == GetGraph()->GetEntryBlock());
313  ComputeSpillMask();
314  first_register_slot_in_slow_path_ = (number_of_out_slots + number_of_spill_slots) * kVRegSize;
315
316  if (number_of_spill_slots == 0
317      && !HasAllocatedCalleeSaveRegisters()
318      && IsLeafMethod()
319      && !RequiresCurrentMethod()) {
320    DCHECK_EQ(maximum_number_of_live_core_registers, 0u);
321    DCHECK_EQ(maximum_number_of_live_fp_registers, 0u);
322    SetFrameSize(CallPushesPC() ? GetWordSize() : 0);
323  } else {
324    SetFrameSize(RoundUp(
325        number_of_spill_slots * kVRegSize
326        + number_of_out_slots * kVRegSize
327        + maximum_number_of_live_core_registers * GetWordSize()
328        + maximum_number_of_live_fp_registers * GetFloatingPointSpillSlotSize()
329        + FrameEntrySpillSize(),
330        kStackAlignment));
331  }
332}
333
334Location CodeGenerator::GetTemporaryLocation(HTemporary* temp) const {
335  uint16_t number_of_locals = GetGraph()->GetNumberOfLocalVRegs();
336  // The type of the previous instruction tells us if we need a single or double stack slot.
337  Primitive::Type type = temp->GetType();
338  int32_t temp_size = (type == Primitive::kPrimLong) || (type == Primitive::kPrimDouble) ? 2 : 1;
339  // Use the temporary region (right below the dex registers).
340  int32_t slot = GetFrameSize() - FrameEntrySpillSize()
341                                - kVRegSize  // filler
342                                - (number_of_locals * kVRegSize)
343                                - ((temp_size + temp->GetIndex()) * kVRegSize);
344  return temp_size == 2 ? Location::DoubleStackSlot(slot) : Location::StackSlot(slot);
345}
346
347int32_t CodeGenerator::GetStackSlot(HLocal* local) const {
348  uint16_t reg_number = local->GetRegNumber();
349  uint16_t number_of_locals = GetGraph()->GetNumberOfLocalVRegs();
350  if (reg_number >= number_of_locals) {
351    // Local is a parameter of the method. It is stored in the caller's frame.
352    // TODO: Share this logic with StackVisitor::GetVRegOffsetFromQuickCode.
353    return GetFrameSize() + InstructionSetPointerSize(GetInstructionSet())  // ART method
354                          + (reg_number - number_of_locals) * kVRegSize;
355  } else {
356    // Local is a temporary in this method. It is stored in this method's frame.
357    return GetFrameSize() - FrameEntrySpillSize()
358                          - kVRegSize  // filler.
359                          - (number_of_locals * kVRegSize)
360                          + (reg_number * kVRegSize);
361  }
362}
363
364void CodeGenerator::CreateCommonInvokeLocationSummary(
365    HInvoke* invoke, InvokeDexCallingConventionVisitor* visitor) {
366  ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena();
367  LocationSummary* locations = new (allocator) LocationSummary(invoke, LocationSummary::kCall);
368
369  for (size_t i = 0; i < invoke->GetNumberOfArguments(); i++) {
370    HInstruction* input = invoke->InputAt(i);
371    locations->SetInAt(i, visitor->GetNextLocation(input->GetType()));
372  }
373
374  locations->SetOut(visitor->GetReturnLocation(invoke->GetType()));
375
376  if (invoke->IsInvokeStaticOrDirect()) {
377    HInvokeStaticOrDirect* call = invoke->AsInvokeStaticOrDirect();
378    if (call->IsStringInit()) {
379      locations->AddTemp(visitor->GetMethodLocation());
380    } else if (call->IsRecursive()) {
381      locations->SetInAt(call->GetCurrentMethodInputIndex(), visitor->GetMethodLocation());
382    } else {
383      locations->AddTemp(visitor->GetMethodLocation());
384      locations->SetInAt(call->GetCurrentMethodInputIndex(), Location::RequiresRegister());
385    }
386  } else {
387    locations->AddTemp(visitor->GetMethodLocation());
388  }
389}
390
391void CodeGenerator::GenerateInvokeUnresolvedRuntimeCall(HInvokeUnresolved* invoke) {
392  MoveConstant(invoke->GetLocations()->GetTemp(0), invoke->GetDexMethodIndex());
393
394  // Initialize to anything to silent compiler warnings.
395  QuickEntrypointEnum entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
396  switch (invoke->GetOriginalInvokeType()) {
397    case kStatic:
398      entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck;
399      break;
400    case kDirect:
401      entrypoint = kQuickInvokeDirectTrampolineWithAccessCheck;
402      break;
403    case kVirtual:
404      entrypoint = kQuickInvokeVirtualTrampolineWithAccessCheck;
405      break;
406    case kSuper:
407      entrypoint = kQuickInvokeSuperTrampolineWithAccessCheck;
408      break;
409    case kInterface:
410      entrypoint = kQuickInvokeInterfaceTrampolineWithAccessCheck;
411      break;
412  }
413  InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), nullptr);
414}
415
416void CodeGenerator::CreateUnresolvedFieldLocationSummary(
417    HInstruction* field_access,
418    Primitive::Type field_type,
419    const FieldAccessCallingConvention& calling_convention) {
420  bool is_instance = field_access->IsUnresolvedInstanceFieldGet()
421      || field_access->IsUnresolvedInstanceFieldSet();
422  bool is_get = field_access->IsUnresolvedInstanceFieldGet()
423      || field_access->IsUnresolvedStaticFieldGet();
424
425  ArenaAllocator* allocator = field_access->GetBlock()->GetGraph()->GetArena();
426  LocationSummary* locations =
427      new (allocator) LocationSummary(field_access, LocationSummary::kCall);
428
429  locations->AddTemp(calling_convention.GetFieldIndexLocation());
430
431  if (is_instance) {
432    // Add the `this` object for instance field accesses.
433    locations->SetInAt(0, calling_convention.GetObjectLocation());
434  }
435
436  // Note that pSetXXStatic/pGetXXStatic always takes/returns an int or int64
437  // regardless of the the type. Because of that we forced to special case
438  // the access to floating point values.
439  if (is_get) {
440    if (Primitive::IsFloatingPointType(field_type)) {
441      // The return value will be stored in regular registers while register
442      // allocator expects it in a floating point register.
443      // Note We don't need to request additional temps because the return
444      // register(s) are already blocked due the call and they may overlap with
445      // the input or field index.
446      // The transfer between the two will be done at codegen level.
447      locations->SetOut(calling_convention.GetFpuLocation(field_type));
448    } else {
449      locations->SetOut(calling_convention.GetReturnLocation(field_type));
450    }
451  } else {
452     size_t set_index = is_instance ? 1 : 0;
453     if (Primitive::IsFloatingPointType(field_type)) {
454      // The set value comes from a float location while the calling convention
455      // expects it in a regular register location. Allocate a temp for it and
456      // make the transfer at codegen.
457      AddLocationAsTemp(calling_convention.GetSetValueLocation(field_type, is_instance), locations);
458      locations->SetInAt(set_index, calling_convention.GetFpuLocation(field_type));
459    } else {
460      locations->SetInAt(set_index,
461          calling_convention.GetSetValueLocation(field_type, is_instance));
462    }
463  }
464}
465
466void CodeGenerator::GenerateUnresolvedFieldAccess(
467    HInstruction* field_access,
468    Primitive::Type field_type,
469    uint32_t field_index,
470    uint32_t dex_pc,
471    const FieldAccessCallingConvention& calling_convention) {
472  LocationSummary* locations = field_access->GetLocations();
473
474  MoveConstant(locations->GetTemp(0), field_index);
475
476  bool is_instance = field_access->IsUnresolvedInstanceFieldGet()
477      || field_access->IsUnresolvedInstanceFieldSet();
478  bool is_get = field_access->IsUnresolvedInstanceFieldGet()
479      || field_access->IsUnresolvedStaticFieldGet();
480
481  if (!is_get && Primitive::IsFloatingPointType(field_type)) {
482    // Copy the float value to be set into the calling convention register.
483    // Note that using directly the temp location is problematic as we don't
484    // support temp register pairs. To avoid boilerplate conversion code, use
485    // the location from the calling convention.
486    MoveLocation(calling_convention.GetSetValueLocation(field_type, is_instance),
487                 locations->InAt(is_instance ? 1 : 0),
488                 (Primitive::Is64BitType(field_type) ? Primitive::kPrimLong : Primitive::kPrimInt));
489  }
490
491  QuickEntrypointEnum entrypoint = kQuickSet8Static;  // Initialize to anything to avoid warnings.
492  switch (field_type) {
493    case Primitive::kPrimBoolean:
494      entrypoint = is_instance
495          ? (is_get ? kQuickGetBooleanInstance : kQuickSet8Instance)
496          : (is_get ? kQuickGetBooleanStatic : kQuickSet8Static);
497      break;
498    case Primitive::kPrimByte:
499      entrypoint = is_instance
500          ? (is_get ? kQuickGetByteInstance : kQuickSet8Instance)
501          : (is_get ? kQuickGetByteStatic : kQuickSet8Static);
502      break;
503    case Primitive::kPrimShort:
504      entrypoint = is_instance
505          ? (is_get ? kQuickGetShortInstance : kQuickSet16Instance)
506          : (is_get ? kQuickGetShortStatic : kQuickSet16Static);
507      break;
508    case Primitive::kPrimChar:
509      entrypoint = is_instance
510          ? (is_get ? kQuickGetCharInstance : kQuickSet16Instance)
511          : (is_get ? kQuickGetCharStatic : kQuickSet16Static);
512      break;
513    case Primitive::kPrimInt:
514    case Primitive::kPrimFloat:
515      entrypoint = is_instance
516          ? (is_get ? kQuickGet32Instance : kQuickSet32Instance)
517          : (is_get ? kQuickGet32Static : kQuickSet32Static);
518      break;
519    case Primitive::kPrimNot:
520      entrypoint = is_instance
521          ? (is_get ? kQuickGetObjInstance : kQuickSetObjInstance)
522          : (is_get ? kQuickGetObjStatic : kQuickSetObjStatic);
523      break;
524    case Primitive::kPrimLong:
525    case Primitive::kPrimDouble:
526      entrypoint = is_instance
527          ? (is_get ? kQuickGet64Instance : kQuickSet64Instance)
528          : (is_get ? kQuickGet64Static : kQuickSet64Static);
529      break;
530    default:
531      LOG(FATAL) << "Invalid type " << field_type;
532  }
533  InvokeRuntime(entrypoint, field_access, dex_pc, nullptr);
534
535  if (is_get && Primitive::IsFloatingPointType(field_type)) {
536    MoveLocation(locations->Out(), calling_convention.GetReturnLocation(field_type), field_type);
537  }
538}
539
540void CodeGenerator::BlockIfInRegister(Location location, bool is_out) const {
541  // The DCHECKS below check that a register is not specified twice in
542  // the summary. The out location can overlap with an input, so we need
543  // to special case it.
544  if (location.IsRegister()) {
545    DCHECK(is_out || !blocked_core_registers_[location.reg()]);
546    blocked_core_registers_[location.reg()] = true;
547  } else if (location.IsFpuRegister()) {
548    DCHECK(is_out || !blocked_fpu_registers_[location.reg()]);
549    blocked_fpu_registers_[location.reg()] = true;
550  } else if (location.IsFpuRegisterPair()) {
551    DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()]);
552    blocked_fpu_registers_[location.AsFpuRegisterPairLow<int>()] = true;
553    DCHECK(is_out || !blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()]);
554    blocked_fpu_registers_[location.AsFpuRegisterPairHigh<int>()] = true;
555  } else if (location.IsRegisterPair()) {
556    DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairLow<int>()]);
557    blocked_core_registers_[location.AsRegisterPairLow<int>()] = true;
558    DCHECK(is_out || !blocked_core_registers_[location.AsRegisterPairHigh<int>()]);
559    blocked_core_registers_[location.AsRegisterPairHigh<int>()] = true;
560  }
561}
562
563void CodeGenerator::AllocateRegistersLocally(HInstruction* instruction) const {
564  LocationSummary* locations = instruction->GetLocations();
565  if (locations == nullptr) return;
566
567  for (size_t i = 0, e = GetNumberOfCoreRegisters(); i < e; ++i) {
568    blocked_core_registers_[i] = false;
569  }
570
571  for (size_t i = 0, e = GetNumberOfFloatingPointRegisters(); i < e; ++i) {
572    blocked_fpu_registers_[i] = false;
573  }
574
575  for (size_t i = 0, e = number_of_register_pairs_; i < e; ++i) {
576    blocked_register_pairs_[i] = false;
577  }
578
579  // Mark all fixed input, temp and output registers as used.
580  for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) {
581    BlockIfInRegister(locations->InAt(i));
582  }
583
584  for (size_t i = 0, e = locations->GetTempCount(); i < e; ++i) {
585    Location loc = locations->GetTemp(i);
586    BlockIfInRegister(loc);
587  }
588  Location result_location = locations->Out();
589  if (locations->OutputCanOverlapWithInputs()) {
590    BlockIfInRegister(result_location, /* is_out */ true);
591  }
592
593  SetupBlockedRegisters(/* is_baseline */ true);
594
595  // Allocate all unallocated input locations.
596  for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) {
597    Location loc = locations->InAt(i);
598    HInstruction* input = instruction->InputAt(i);
599    if (loc.IsUnallocated()) {
600      if ((loc.GetPolicy() == Location::kRequiresRegister)
601          || (loc.GetPolicy() == Location::kRequiresFpuRegister)) {
602        loc = AllocateFreeRegister(input->GetType());
603      } else {
604        DCHECK_EQ(loc.GetPolicy(), Location::kAny);
605        HLoadLocal* load = input->AsLoadLocal();
606        if (load != nullptr) {
607          loc = GetStackLocation(load);
608        } else {
609          loc = AllocateFreeRegister(input->GetType());
610        }
611      }
612      locations->SetInAt(i, loc);
613    }
614  }
615
616  // Allocate all unallocated temp locations.
617  for (size_t i = 0, e = locations->GetTempCount(); i < e; ++i) {
618    Location loc = locations->GetTemp(i);
619    if (loc.IsUnallocated()) {
620      switch (loc.GetPolicy()) {
621        case Location::kRequiresRegister:
622          // Allocate a core register (large enough to fit a 32-bit integer).
623          loc = AllocateFreeRegister(Primitive::kPrimInt);
624          break;
625
626        case Location::kRequiresFpuRegister:
627          // Allocate a core register (large enough to fit a 64-bit double).
628          loc = AllocateFreeRegister(Primitive::kPrimDouble);
629          break;
630
631        default:
632          LOG(FATAL) << "Unexpected policy for temporary location "
633                     << loc.GetPolicy();
634      }
635      locations->SetTempAt(i, loc);
636    }
637  }
638  if (result_location.IsUnallocated()) {
639    switch (result_location.GetPolicy()) {
640      case Location::kAny:
641      case Location::kRequiresRegister:
642      case Location::kRequiresFpuRegister:
643        result_location = AllocateFreeRegister(instruction->GetType());
644        break;
645      case Location::kSameAsFirstInput:
646        result_location = locations->InAt(0);
647        break;
648    }
649    locations->UpdateOut(result_location);
650  }
651}
652
653void CodeGenerator::InitLocationsBaseline(HInstruction* instruction) {
654  AllocateLocations(instruction);
655  if (instruction->GetLocations() == nullptr) {
656    if (instruction->IsTemporary()) {
657      HInstruction* previous = instruction->GetPrevious();
658      Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
659      Move(previous, temp_location, instruction);
660    }
661    return;
662  }
663  AllocateRegistersLocally(instruction);
664  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
665    Location location = instruction->GetLocations()->InAt(i);
666    HInstruction* input = instruction->InputAt(i);
667    if (location.IsValid()) {
668      // Move the input to the desired location.
669      if (input->GetNext()->IsTemporary()) {
670        // If the input was stored in a temporary, use that temporary to
671        // perform the move.
672        Move(input->GetNext(), location, instruction);
673      } else {
674        Move(input, location, instruction);
675      }
676    }
677  }
678}
679
680void CodeGenerator::AllocateLocations(HInstruction* instruction) {
681  instruction->Accept(GetLocationBuilder());
682  DCHECK(CheckTypeConsistency(instruction));
683  LocationSummary* locations = instruction->GetLocations();
684  if (!instruction->IsSuspendCheckEntry()) {
685    if (locations != nullptr && locations->CanCall()) {
686      MarkNotLeaf();
687    }
688    if (instruction->NeedsCurrentMethod()) {
689      SetRequiresCurrentMethod();
690    }
691  }
692}
693
694void CodeGenerator::MaybeRecordStat(MethodCompilationStat compilation_stat, size_t count) const {
695  if (stats_ != nullptr) {
696    stats_->RecordStat(compilation_stat, count);
697  }
698}
699
700CodeGenerator* CodeGenerator::Create(HGraph* graph,
701                                     InstructionSet instruction_set,
702                                     const InstructionSetFeatures& isa_features,
703                                     const CompilerOptions& compiler_options,
704                                     OptimizingCompilerStats* stats) {
705  switch (instruction_set) {
706#ifdef ART_ENABLE_CODEGEN_arm
707    case kArm:
708    case kThumb2: {
709      return new arm::CodeGeneratorARM(graph,
710                                      *isa_features.AsArmInstructionSetFeatures(),
711                                      compiler_options,
712                                      stats);
713    }
714#endif
715#ifdef ART_ENABLE_CODEGEN_arm64
716    case kArm64: {
717      return new arm64::CodeGeneratorARM64(graph,
718                                          *isa_features.AsArm64InstructionSetFeatures(),
719                                          compiler_options,
720                                          stats);
721    }
722#endif
723#ifdef ART_ENABLE_CODEGEN_mips
724    case kMips:
725      UNUSED(compiler_options);
726      UNUSED(graph);
727      UNUSED(isa_features);
728      return nullptr;
729#endif
730#ifdef ART_ENABLE_CODEGEN_mips64
731    case kMips64: {
732      return new mips64::CodeGeneratorMIPS64(graph,
733                                            *isa_features.AsMips64InstructionSetFeatures(),
734                                            compiler_options,
735                                            stats);
736    }
737#endif
738#ifdef ART_ENABLE_CODEGEN_x86
739    case kX86: {
740      return new x86::CodeGeneratorX86(graph,
741                                      *isa_features.AsX86InstructionSetFeatures(),
742                                      compiler_options,
743                                      stats);
744    }
745#endif
746#ifdef ART_ENABLE_CODEGEN_x86_64
747    case kX86_64: {
748      return new x86_64::CodeGeneratorX86_64(graph,
749                                            *isa_features.AsX86_64InstructionSetFeatures(),
750                                            compiler_options,
751                                            stats);
752    }
753#endif
754    default:
755      return nullptr;
756  }
757}
758
759void CodeGenerator::BuildNativeGCMap(
760    ArenaVector<uint8_t>* data, const DexCompilationUnit& dex_compilation_unit) const {
761  const std::vector<uint8_t>& gc_map_raw =
762      dex_compilation_unit.GetVerifiedMethod()->GetDexGcMap();
763  verifier::DexPcToReferenceMap dex_gc_map(&(gc_map_raw)[0]);
764
765  uint32_t max_native_offset = stack_map_stream_.ComputeMaxNativePcOffset();
766
767  size_t num_stack_maps = stack_map_stream_.GetNumberOfStackMaps();
768  GcMapBuilder builder(data, num_stack_maps, max_native_offset, dex_gc_map.RegWidth());
769  for (size_t i = 0; i != num_stack_maps; ++i) {
770    const StackMapStream::StackMapEntry& stack_map_entry = stack_map_stream_.GetStackMap(i);
771    uint32_t native_offset = stack_map_entry.native_pc_offset;
772    uint32_t dex_pc = stack_map_entry.dex_pc;
773    const uint8_t* references = dex_gc_map.FindBitMap(dex_pc, false);
774    CHECK(references != nullptr) << "Missing ref for dex pc 0x" << std::hex << dex_pc;
775    builder.AddEntry(native_offset, references);
776  }
777}
778
779void CodeGenerator::BuildMappingTable(ArenaVector<uint8_t>* data) const {
780  uint32_t pc2dex_data_size = 0u;
781  uint32_t pc2dex_entries = stack_map_stream_.GetNumberOfStackMaps();
782  uint32_t pc2dex_offset = 0u;
783  int32_t pc2dex_dalvik_offset = 0;
784  uint32_t dex2pc_data_size = 0u;
785  uint32_t dex2pc_entries = 0u;
786  uint32_t dex2pc_offset = 0u;
787  int32_t dex2pc_dalvik_offset = 0;
788
789  for (size_t i = 0; i < pc2dex_entries; i++) {
790    const StackMapStream::StackMapEntry& stack_map_entry = stack_map_stream_.GetStackMap(i);
791    pc2dex_data_size += UnsignedLeb128Size(stack_map_entry.native_pc_offset - pc2dex_offset);
792    pc2dex_data_size += SignedLeb128Size(stack_map_entry.dex_pc - pc2dex_dalvik_offset);
793    pc2dex_offset = stack_map_entry.native_pc_offset;
794    pc2dex_dalvik_offset = stack_map_entry.dex_pc;
795  }
796
797  // Walk over the blocks and find which ones correspond to catch block entries.
798  for (HBasicBlock* block : graph_->GetBlocks()) {
799    if (block->IsCatchBlock()) {
800      intptr_t native_pc = GetAddressOf(block);
801      ++dex2pc_entries;
802      dex2pc_data_size += UnsignedLeb128Size(native_pc - dex2pc_offset);
803      dex2pc_data_size += SignedLeb128Size(block->GetDexPc() - dex2pc_dalvik_offset);
804      dex2pc_offset = native_pc;
805      dex2pc_dalvik_offset = block->GetDexPc();
806    }
807  }
808
809  uint32_t total_entries = pc2dex_entries + dex2pc_entries;
810  uint32_t hdr_data_size = UnsignedLeb128Size(total_entries) + UnsignedLeb128Size(pc2dex_entries);
811  uint32_t data_size = hdr_data_size + pc2dex_data_size + dex2pc_data_size;
812  data->resize(data_size);
813
814  uint8_t* data_ptr = &(*data)[0];
815  uint8_t* write_pos = data_ptr;
816
817  write_pos = EncodeUnsignedLeb128(write_pos, total_entries);
818  write_pos = EncodeUnsignedLeb128(write_pos, pc2dex_entries);
819  DCHECK_EQ(static_cast<size_t>(write_pos - data_ptr), hdr_data_size);
820  uint8_t* write_pos2 = write_pos + pc2dex_data_size;
821
822  pc2dex_offset = 0u;
823  pc2dex_dalvik_offset = 0u;
824  dex2pc_offset = 0u;
825  dex2pc_dalvik_offset = 0u;
826
827  for (size_t i = 0; i < pc2dex_entries; i++) {
828    const StackMapStream::StackMapEntry& stack_map_entry = stack_map_stream_.GetStackMap(i);
829    DCHECK(pc2dex_offset <= stack_map_entry.native_pc_offset);
830    write_pos = EncodeUnsignedLeb128(write_pos, stack_map_entry.native_pc_offset - pc2dex_offset);
831    write_pos = EncodeSignedLeb128(write_pos, stack_map_entry.dex_pc - pc2dex_dalvik_offset);
832    pc2dex_offset = stack_map_entry.native_pc_offset;
833    pc2dex_dalvik_offset = stack_map_entry.dex_pc;
834  }
835
836  for (HBasicBlock* block : graph_->GetBlocks()) {
837    if (block->IsCatchBlock()) {
838      intptr_t native_pc = GetAddressOf(block);
839      write_pos2 = EncodeUnsignedLeb128(write_pos2, native_pc - dex2pc_offset);
840      write_pos2 = EncodeSignedLeb128(write_pos2, block->GetDexPc() - dex2pc_dalvik_offset);
841      dex2pc_offset = native_pc;
842      dex2pc_dalvik_offset = block->GetDexPc();
843    }
844  }
845
846
847  DCHECK_EQ(static_cast<size_t>(write_pos - data_ptr), hdr_data_size + pc2dex_data_size);
848  DCHECK_EQ(static_cast<size_t>(write_pos2 - data_ptr), data_size);
849
850  if (kIsDebugBuild) {
851    // Verify the encoded table holds the expected data.
852    MappingTable table(data_ptr);
853    CHECK_EQ(table.TotalSize(), total_entries);
854    CHECK_EQ(table.PcToDexSize(), pc2dex_entries);
855    auto it = table.PcToDexBegin();
856    auto it2 = table.DexToPcBegin();
857    for (size_t i = 0; i < pc2dex_entries; i++) {
858      const StackMapStream::StackMapEntry& stack_map_entry = stack_map_stream_.GetStackMap(i);
859      CHECK_EQ(stack_map_entry.native_pc_offset, it.NativePcOffset());
860      CHECK_EQ(stack_map_entry.dex_pc, it.DexPc());
861      ++it;
862    }
863    for (HBasicBlock* block : graph_->GetBlocks()) {
864      if (block->IsCatchBlock()) {
865        CHECK_EQ(GetAddressOf(block), it2.NativePcOffset());
866        CHECK_EQ(block->GetDexPc(), it2.DexPc());
867        ++it2;
868      }
869    }
870    CHECK(it == table.PcToDexEnd());
871    CHECK(it2 == table.DexToPcEnd());
872  }
873}
874
875void CodeGenerator::BuildVMapTable(ArenaVector<uint8_t>* data) const {
876  Leb128Encoder<ArenaAllocatorAdapter<uint8_t>> vmap_encoder(data);
877  // We currently don't use callee-saved registers.
878  size_t size = 0 + 1 /* marker */ + 0;
879  vmap_encoder.Reserve(size + 1u);  // All values are likely to be one byte in ULEB128 (<128).
880  vmap_encoder.PushBackUnsigned(size);
881  vmap_encoder.PushBackUnsigned(VmapTable::kAdjustedFpMarker);
882}
883
884void CodeGenerator::BuildStackMaps(ArenaVector<uint8_t>* data) {
885  uint32_t size = stack_map_stream_.PrepareForFillIn();
886  data->resize(size);
887  MemoryRegion region(data->data(), size);
888  stack_map_stream_.FillIn(region);
889}
890
891void CodeGenerator::RecordNativeDebugInfo(uint32_t dex_pc,
892                                          uintptr_t native_pc_begin,
893                                          uintptr_t native_pc_end) {
894  if (src_map_ != nullptr && dex_pc != kNoDexPc && native_pc_begin != native_pc_end) {
895    src_map_->push_back(SrcMapElem({static_cast<uint32_t>(native_pc_begin),
896                                    static_cast<int32_t>(dex_pc)}));
897  }
898}
899
900void CodeGenerator::RecordPcInfo(HInstruction* instruction,
901                                 uint32_t dex_pc,
902                                 SlowPathCode* slow_path) {
903  if (instruction != nullptr) {
904    // The code generated for some type conversions and comparisons
905    // may call the runtime, thus normally requiring a subsequent
906    // call to this method. However, the method verifier does not
907    // produce PC information for certain instructions, which are
908    // considered "atomic" (they cannot join a GC).
909    // Therefore we do not currently record PC information for such
910    // instructions.  As this may change later, we added this special
911    // case so that code generators may nevertheless call
912    // CodeGenerator::RecordPcInfo without triggering an error in
913    // CodeGenerator::BuildNativeGCMap ("Missing ref for dex pc 0x")
914    // thereafter.
915    if (instruction->IsTypeConversion() || instruction->IsCompare()) {
916      return;
917    }
918    if (instruction->IsRem()) {
919      Primitive::Type type = instruction->AsRem()->GetResultType();
920      if ((type == Primitive::kPrimFloat) || (type == Primitive::kPrimDouble)) {
921        return;
922      }
923    }
924  }
925
926  uint32_t outer_dex_pc = dex_pc;
927  uint32_t outer_environment_size = 0;
928  uint32_t inlining_depth = 0;
929  if (instruction != nullptr) {
930    for (HEnvironment* environment = instruction->GetEnvironment();
931         environment != nullptr;
932         environment = environment->GetParent()) {
933      outer_dex_pc = environment->GetDexPc();
934      outer_environment_size = environment->Size();
935      if (environment != instruction->GetEnvironment()) {
936        inlining_depth++;
937      }
938    }
939  }
940
941  // Collect PC infos for the mapping table.
942  uint32_t native_pc = GetAssembler()->CodeSize();
943
944  if (instruction == nullptr) {
945    // For stack overflow checks.
946    stack_map_stream_.BeginStackMapEntry(outer_dex_pc, native_pc, 0, 0, 0, 0);
947    stack_map_stream_.EndStackMapEntry();
948    return;
949  }
950  LocationSummary* locations = instruction->GetLocations();
951
952  uint32_t register_mask = locations->GetRegisterMask();
953  if (locations->OnlyCallsOnSlowPath()) {
954    // In case of slow path, we currently set the location of caller-save registers
955    // to register (instead of their stack location when pushed before the slow-path
956    // call). Therefore register_mask contains both callee-save and caller-save
957    // registers that hold objects. We must remove the caller-save from the mask, since
958    // they will be overwritten by the callee.
959    register_mask &= core_callee_save_mask_;
960  }
961  // The register mask must be a subset of callee-save registers.
962  DCHECK_EQ(register_mask & core_callee_save_mask_, register_mask);
963  stack_map_stream_.BeginStackMapEntry(outer_dex_pc,
964                                       native_pc,
965                                       register_mask,
966                                       locations->GetStackMask(),
967                                       outer_environment_size,
968                                       inlining_depth);
969
970  EmitEnvironment(instruction->GetEnvironment(), slow_path);
971  stack_map_stream_.EndStackMapEntry();
972}
973
974void CodeGenerator::RecordCatchBlockInfo() {
975  ArenaAllocator* arena = graph_->GetArena();
976
977  for (HBasicBlock* block : *block_order_) {
978    if (!block->IsCatchBlock()) {
979      continue;
980    }
981
982    uint32_t dex_pc = block->GetDexPc();
983    uint32_t num_vregs = graph_->GetNumberOfVRegs();
984    uint32_t inlining_depth = 0;  // Inlining of catch blocks is not supported at the moment.
985    uint32_t native_pc = GetAddressOf(block);
986    uint32_t register_mask = 0;   // Not used.
987
988    // The stack mask is not used, so we leave it empty.
989    ArenaBitVector* stack_mask = new (arena) ArenaBitVector(arena, 0, /* expandable */ true);
990
991    stack_map_stream_.BeginStackMapEntry(dex_pc,
992                                         native_pc,
993                                         register_mask,
994                                         stack_mask,
995                                         num_vregs,
996                                         inlining_depth);
997
998    HInstruction* current_phi = block->GetFirstPhi();
999    for (size_t vreg = 0; vreg < num_vregs; ++vreg) {
1000    while (current_phi != nullptr && current_phi->AsPhi()->GetRegNumber() < vreg) {
1001      HInstruction* next_phi = current_phi->GetNext();
1002      DCHECK(next_phi == nullptr ||
1003             current_phi->AsPhi()->GetRegNumber() <= next_phi->AsPhi()->GetRegNumber())
1004          << "Phis need to be sorted by vreg number to keep this a linear-time loop.";
1005      current_phi = next_phi;
1006    }
1007
1008      if (current_phi == nullptr || current_phi->AsPhi()->GetRegNumber() != vreg) {
1009        stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
1010      } else {
1011        Location location = current_phi->GetLiveInterval()->ToLocation();
1012        switch (location.GetKind()) {
1013          case Location::kStackSlot: {
1014            stack_map_stream_.AddDexRegisterEntry(
1015                DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
1016            break;
1017          }
1018          case Location::kDoubleStackSlot: {
1019            stack_map_stream_.AddDexRegisterEntry(
1020                DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
1021            stack_map_stream_.AddDexRegisterEntry(
1022                DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize));
1023            ++vreg;
1024            DCHECK_LT(vreg, num_vregs);
1025            break;
1026          }
1027          default: {
1028            // All catch phis must be allocated to a stack slot.
1029            LOG(FATAL) << "Unexpected kind " << location.GetKind();
1030            UNREACHABLE();
1031          }
1032        }
1033      }
1034    }
1035
1036    stack_map_stream_.EndStackMapEntry();
1037  }
1038}
1039
1040void CodeGenerator::EmitEnvironment(HEnvironment* environment, SlowPathCode* slow_path) {
1041  if (environment == nullptr) return;
1042
1043  if (environment->GetParent() != nullptr) {
1044    // We emit the parent environment first.
1045    EmitEnvironment(environment->GetParent(), slow_path);
1046    stack_map_stream_.BeginInlineInfoEntry(environment->GetMethodIdx(),
1047                                           environment->GetDexPc(),
1048                                           environment->GetInvokeType(),
1049                                           environment->Size());
1050  }
1051
1052  // Walk over the environment, and record the location of dex registers.
1053  for (size_t i = 0, environment_size = environment->Size(); i < environment_size; ++i) {
1054    HInstruction* current = environment->GetInstructionAt(i);
1055    if (current == nullptr) {
1056      stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
1057      continue;
1058    }
1059
1060    Location location = environment->GetLocationAt(i);
1061    switch (location.GetKind()) {
1062      case Location::kConstant: {
1063        DCHECK_EQ(current, location.GetConstant());
1064        if (current->IsLongConstant()) {
1065          int64_t value = current->AsLongConstant()->GetValue();
1066          stack_map_stream_.AddDexRegisterEntry(
1067              DexRegisterLocation::Kind::kConstant, Low32Bits(value));
1068          stack_map_stream_.AddDexRegisterEntry(
1069              DexRegisterLocation::Kind::kConstant, High32Bits(value));
1070          ++i;
1071          DCHECK_LT(i, environment_size);
1072        } else if (current->IsDoubleConstant()) {
1073          int64_t value = bit_cast<int64_t, double>(current->AsDoubleConstant()->GetValue());
1074          stack_map_stream_.AddDexRegisterEntry(
1075              DexRegisterLocation::Kind::kConstant, Low32Bits(value));
1076          stack_map_stream_.AddDexRegisterEntry(
1077              DexRegisterLocation::Kind::kConstant, High32Bits(value));
1078          ++i;
1079          DCHECK_LT(i, environment_size);
1080        } else if (current->IsIntConstant()) {
1081          int32_t value = current->AsIntConstant()->GetValue();
1082          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, value);
1083        } else if (current->IsNullConstant()) {
1084          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, 0);
1085        } else {
1086          DCHECK(current->IsFloatConstant()) << current->DebugName();
1087          int32_t value = bit_cast<int32_t, float>(current->AsFloatConstant()->GetValue());
1088          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kConstant, value);
1089        }
1090        break;
1091      }
1092
1093      case Location::kStackSlot: {
1094        stack_map_stream_.AddDexRegisterEntry(
1095            DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
1096        break;
1097      }
1098
1099      case Location::kDoubleStackSlot: {
1100        stack_map_stream_.AddDexRegisterEntry(
1101            DexRegisterLocation::Kind::kInStack, location.GetStackIndex());
1102        stack_map_stream_.AddDexRegisterEntry(
1103            DexRegisterLocation::Kind::kInStack, location.GetHighStackIndex(kVRegSize));
1104        ++i;
1105        DCHECK_LT(i, environment_size);
1106        break;
1107      }
1108
1109      case Location::kRegister : {
1110        int id = location.reg();
1111        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(id)) {
1112          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(id);
1113          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1114          if (current->GetType() == Primitive::kPrimLong) {
1115            stack_map_stream_.AddDexRegisterEntry(
1116                DexRegisterLocation::Kind::kInStack, offset + kVRegSize);
1117            ++i;
1118            DCHECK_LT(i, environment_size);
1119          }
1120        } else {
1121          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, id);
1122          if (current->GetType() == Primitive::kPrimLong) {
1123            stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegisterHigh, id);
1124            ++i;
1125            DCHECK_LT(i, environment_size);
1126          }
1127        }
1128        break;
1129      }
1130
1131      case Location::kFpuRegister : {
1132        int id = location.reg();
1133        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(id)) {
1134          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(id);
1135          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1136          if (current->GetType() == Primitive::kPrimDouble) {
1137            stack_map_stream_.AddDexRegisterEntry(
1138                DexRegisterLocation::Kind::kInStack, offset + kVRegSize);
1139            ++i;
1140            DCHECK_LT(i, environment_size);
1141          }
1142        } else {
1143          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, id);
1144          if (current->GetType() == Primitive::kPrimDouble) {
1145            stack_map_stream_.AddDexRegisterEntry(
1146                DexRegisterLocation::Kind::kInFpuRegisterHigh, id);
1147            ++i;
1148            DCHECK_LT(i, environment_size);
1149          }
1150        }
1151        break;
1152      }
1153
1154      case Location::kFpuRegisterPair : {
1155        int low = location.low();
1156        int high = location.high();
1157        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(low)) {
1158          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(low);
1159          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1160        } else {
1161          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, low);
1162        }
1163        if (slow_path != nullptr && slow_path->IsFpuRegisterSaved(high)) {
1164          uint32_t offset = slow_path->GetStackOffsetOfFpuRegister(high);
1165          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1166          ++i;
1167        } else {
1168          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInFpuRegister, high);
1169          ++i;
1170        }
1171        DCHECK_LT(i, environment_size);
1172        break;
1173      }
1174
1175      case Location::kRegisterPair : {
1176        int low = location.low();
1177        int high = location.high();
1178        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(low)) {
1179          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(low);
1180          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1181        } else {
1182          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, low);
1183        }
1184        if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(high)) {
1185          uint32_t offset = slow_path->GetStackOffsetOfCoreRegister(high);
1186          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInStack, offset);
1187        } else {
1188          stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kInRegister, high);
1189        }
1190        ++i;
1191        DCHECK_LT(i, environment_size);
1192        break;
1193      }
1194
1195      case Location::kInvalid: {
1196        stack_map_stream_.AddDexRegisterEntry(DexRegisterLocation::Kind::kNone, 0);
1197        break;
1198      }
1199
1200      default:
1201        LOG(FATAL) << "Unexpected kind " << location.GetKind();
1202    }
1203  }
1204
1205  if (environment->GetParent() != nullptr) {
1206    stack_map_stream_.EndInlineInfoEntry();
1207  }
1208}
1209
1210bool CodeGenerator::IsImplicitNullCheckAllowed(HNullCheck* null_check) const {
1211  return compiler_options_.GetImplicitNullChecks() &&
1212         // Null checks which might throw into a catch block need to save live
1213         // registers and therefore cannot be done implicitly.
1214         !null_check->CanThrowIntoCatchBlock();
1215}
1216
1217bool CodeGenerator::CanMoveNullCheckToUser(HNullCheck* null_check) {
1218  HInstruction* first_next_not_move = null_check->GetNextDisregardingMoves();
1219
1220  return (first_next_not_move != nullptr)
1221      && first_next_not_move->CanDoImplicitNullCheckOn(null_check->InputAt(0));
1222}
1223
1224void CodeGenerator::MaybeRecordImplicitNullCheck(HInstruction* instr) {
1225  // If we are from a static path don't record the pc as we can't throw NPE.
1226  // NB: having the checks here makes the code much less verbose in the arch
1227  // specific code generators.
1228  if (instr->IsStaticFieldSet() || instr->IsStaticFieldGet()) {
1229    return;
1230  }
1231
1232  if (!instr->CanDoImplicitNullCheckOn(instr->InputAt(0))) {
1233    return;
1234  }
1235
1236  // Find the first previous instruction which is not a move.
1237  HInstruction* first_prev_not_move = instr->GetPreviousDisregardingMoves();
1238
1239  // If the instruction is a null check it means that `instr` is the first user
1240  // and needs to record the pc.
1241  if (first_prev_not_move != nullptr && first_prev_not_move->IsNullCheck()) {
1242    HNullCheck* null_check = first_prev_not_move->AsNullCheck();
1243    if (IsImplicitNullCheckAllowed(null_check)) {
1244      // TODO: The parallel moves modify the environment. Their changes need to be
1245      // reverted otherwise the stack maps at the throw point will not be correct.
1246      RecordPcInfo(null_check, null_check->GetDexPc());
1247    }
1248  }
1249}
1250
1251void CodeGenerator::ClearSpillSlotsFromLoopPhisInStackMap(HSuspendCheck* suspend_check) const {
1252  LocationSummary* locations = suspend_check->GetLocations();
1253  HBasicBlock* block = suspend_check->GetBlock();
1254  DCHECK(block->GetLoopInformation()->GetSuspendCheck() == suspend_check);
1255  DCHECK(block->IsLoopHeader());
1256
1257  for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) {
1258    HInstruction* current = it.Current();
1259    LiveInterval* interval = current->GetLiveInterval();
1260    // We only need to clear bits of loop phis containing objects and allocated in register.
1261    // Loop phis allocated on stack already have the object in the stack.
1262    if (current->GetType() == Primitive::kPrimNot
1263        && interval->HasRegister()
1264        && interval->HasSpillSlot()) {
1265      locations->ClearStackBit(interval->GetSpillSlot() / kVRegSize);
1266    }
1267  }
1268}
1269
1270void CodeGenerator::EmitParallelMoves(Location from1,
1271                                      Location to1,
1272                                      Primitive::Type type1,
1273                                      Location from2,
1274                                      Location to2,
1275                                      Primitive::Type type2) {
1276  HParallelMove parallel_move(GetGraph()->GetArena());
1277  parallel_move.AddMove(from1, to1, type1, nullptr);
1278  parallel_move.AddMove(from2, to2, type2, nullptr);
1279  GetMoveResolver()->EmitNativeCode(&parallel_move);
1280}
1281
1282void CodeGenerator::ValidateInvokeRuntime(HInstruction* instruction, SlowPathCode* slow_path) {
1283  // Ensure that the call kind indication given to the register allocator is
1284  // coherent with the runtime call generated, and that the GC side effect is
1285  // set when required.
1286  if (slow_path == nullptr) {
1287    DCHECK(instruction->GetLocations()->WillCall()) << instruction->DebugName();
1288    DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()))
1289        << instruction->DebugName() << instruction->GetSideEffects().ToString();
1290  } else {
1291    DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath() || slow_path->IsFatal())
1292        << instruction->DebugName() << slow_path->GetDescription();
1293    DCHECK(instruction->GetSideEffects().Includes(SideEffects::CanTriggerGC()) ||
1294           // Control flow would not come back into the code if a fatal slow
1295           // path is taken, so we do not care if it triggers GC.
1296           slow_path->IsFatal() ||
1297           // HDeoptimize is a special case: we know we are not coming back from
1298           // it into the code.
1299           instruction->IsDeoptimize())
1300        << instruction->DebugName() << instruction->GetSideEffects().ToString()
1301        << slow_path->GetDescription();
1302  }
1303
1304  // Check the coherency of leaf information.
1305  DCHECK(instruction->IsSuspendCheck()
1306         || ((slow_path != nullptr) && slow_path->IsFatal())
1307         || instruction->GetLocations()->CanCall()
1308         || !IsLeafMethod())
1309      << instruction->DebugName() << ((slow_path != nullptr) ? slow_path->GetDescription() : "");
1310}
1311
1312void SlowPathCode::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
1313  RegisterSet* register_set = locations->GetLiveRegisters();
1314  size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
1315  for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
1316    if (!codegen->IsCoreCalleeSaveRegister(i)) {
1317      if (register_set->ContainsCoreRegister(i)) {
1318        // If the register holds an object, update the stack mask.
1319        if (locations->RegisterContainsObject(i)) {
1320          locations->SetStackBit(stack_offset / kVRegSize);
1321        }
1322        DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1323        DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1324        saved_core_stack_offsets_[i] = stack_offset;
1325        stack_offset += codegen->SaveCoreRegister(stack_offset, i);
1326      }
1327    }
1328  }
1329
1330  for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) {
1331    if (!codegen->IsFloatingPointCalleeSaveRegister(i)) {
1332      if (register_set->ContainsFloatingPointRegister(i)) {
1333        DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1334        DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
1335        saved_fpu_stack_offsets_[i] = stack_offset;
1336        stack_offset += codegen->SaveFloatingPointRegister(stack_offset, i);
1337      }
1338    }
1339  }
1340}
1341
1342void SlowPathCode::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
1343  RegisterSet* register_set = locations->GetLiveRegisters();
1344  size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
1345  for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
1346    if (!codegen->IsCoreCalleeSaveRegister(i)) {
1347      if (register_set->ContainsCoreRegister(i)) {
1348        DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1349        stack_offset += codegen->RestoreCoreRegister(stack_offset, i);
1350      }
1351    }
1352  }
1353
1354  for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) {
1355    if (!codegen->IsFloatingPointCalleeSaveRegister(i)) {
1356      if (register_set->ContainsFloatingPointRegister(i)) {
1357        DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
1358        stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, i);
1359      }
1360    }
1361  }
1362}
1363
1364}  // namespace art
1365