1b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko/* 2b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko * Copyright (C) 2015 The Android Open Source Project 3b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko * 4b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko * Licensed under the Apache License, Version 2.0 (the "License"); 5b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko * you may not use this file except in compliance with the License. 6b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko * You may obtain a copy of the License at 7b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko * 8b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko * http://www.apache.org/licenses/LICENSE-2.0 9b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko * 10b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko * Unless required by applicable law or agreed to in writing, software 11b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko * distributed under the License is distributed on an "AS IS" BASIS, 12b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko * See the License for the specific language governing permissions and 14b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko * limitations under the License. 15b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko */ 16b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 17b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko#include "linker/arm64/relative_patcher_arm64.h" 18b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 19b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko#include "arch/arm64/instruction_set_features_arm64.h" 203d21bdf8894e780d349c481e5c9e29fe1556051cMathieu Chartier#include "art_method.h" 21b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko#include "compiled_method.h" 22b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko#include "driver/compiler_driver.h" 23b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko#include "utils/arm64/assembler_arm64.h" 24b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko#include "oat.h" 25b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko#include "output_stream.h" 26b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 27b163bb742a099c1808907b513ae39068b63b1692Vladimir Markonamespace art { 28b163bb742a099c1808907b513ae39068b63b1692Vladimir Markonamespace linker { 29b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 30b163bb742a099c1808907b513ae39068b63b1692Vladimir MarkoArm64RelativePatcher::Arm64RelativePatcher(RelativePatcherTargetProvider* provider, 31b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko const Arm64InstructionSetFeatures* features) 32b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko : ArmBaseRelativePatcher(provider, kArm64, CompileThunkCode(), 33b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko kMaxPositiveDisplacement, kMaxNegativeDisplacement), 34b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko fix_cortex_a53_843419_(features->NeedFixCortexA53_843419()), 35b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko reserved_adrp_thunks_(0u), 36b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko processed_adrp_thunks_(0u) { 37b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (fix_cortex_a53_843419_) { 38b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko adrp_thunk_locations_.reserve(16u); 39b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko current_method_thunks_.reserve(16u * kAdrpThunkSize); 40b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 41b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko} 42b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 43b163bb742a099c1808907b513ae39068b63b1692Vladimir Markouint32_t Arm64RelativePatcher::ReserveSpace(uint32_t offset, 444d23c9d01b7a609813345eec95167a4dbc4fbae4Vladimir Marko const CompiledMethod* compiled_method, 454d23c9d01b7a609813345eec95167a4dbc4fbae4Vladimir Marko MethodReference method_ref) { 46b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (!fix_cortex_a53_843419_) { 47b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK(adrp_thunk_locations_.empty()); 484d23c9d01b7a609813345eec95167a4dbc4fbae4Vladimir Marko return ReserveSpaceInternal(offset, compiled_method, method_ref, 0u); 49b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 50b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 51b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // Add thunks for previous method if any. 52b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (reserved_adrp_thunks_ != adrp_thunk_locations_.size()) { 53b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko size_t num_adrp_thunks = adrp_thunk_locations_.size() - reserved_adrp_thunks_; 54b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko offset = CompiledMethod::AlignCode(offset, kArm64) + kAdrpThunkSize * num_adrp_thunks; 55b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko reserved_adrp_thunks_ = adrp_thunk_locations_.size(); 56b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 57b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 58b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // Count the number of ADRP insns as the upper bound on the number of thunks needed 59b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // and use it to reserve space for other linker patches. 60b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko size_t num_adrp = 0u; 6171b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko DCHECK(compiled_method != nullptr); 6271b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko for (const LinkerPatch& patch : compiled_method->GetPatches()) { 6371b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko if (patch.Type() == kLinkerPatchDexCacheArray && 6471b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko patch.LiteralOffset() == patch.PcInsnOffset()) { // ADRP patch 6571b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko ++num_adrp; 66b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 67b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 684d23c9d01b7a609813345eec95167a4dbc4fbae4Vladimir Marko offset = ReserveSpaceInternal(offset, compiled_method, method_ref, kAdrpThunkSize * num_adrp); 69b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (num_adrp == 0u) { 70b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko return offset; 71b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 72b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 73b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // Now that we have the actual offset where the code will be placed, locate the ADRP insns 74b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // that actually require the thunk. 75b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t quick_code_offset = compiled_method->AlignCode(offset) + sizeof(OatQuickMethodHeader); 76b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko ArrayRef<const uint8_t> code(*compiled_method->GetQuickCode()); 77b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t thunk_offset = compiled_method->AlignCode(quick_code_offset + code.size()); 78b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK(compiled_method != nullptr); 79b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko for (const LinkerPatch& patch : compiled_method->GetPatches()) { 80b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (patch.Type() == kLinkerPatchDexCacheArray && 81b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko patch.LiteralOffset() == patch.PcInsnOffset()) { // ADRP patch 82b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t patch_offset = quick_code_offset + patch.LiteralOffset(); 83b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (NeedsErratum843419Thunk(code, patch.LiteralOffset(), patch_offset)) { 84b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko adrp_thunk_locations_.emplace_back(patch_offset, thunk_offset); 85b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko thunk_offset += kAdrpThunkSize; 86b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 87b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 88b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 89b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko return offset; 90b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko} 91b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 9271b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Markouint32_t Arm64RelativePatcher::ReserveSpaceEnd(uint32_t offset) { 9371b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko if (!fix_cortex_a53_843419_) { 9471b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko DCHECK(adrp_thunk_locations_.empty()); 9571b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko } else { 9671b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko // Add thunks for the last method if any. 9771b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko if (reserved_adrp_thunks_ != adrp_thunk_locations_.size()) { 9871b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko size_t num_adrp_thunks = adrp_thunk_locations_.size() - reserved_adrp_thunks_; 9971b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko offset = CompiledMethod::AlignCode(offset, kArm64) + kAdrpThunkSize * num_adrp_thunks; 10071b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko reserved_adrp_thunks_ = adrp_thunk_locations_.size(); 10171b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko } 10271b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko } 10371b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko return ArmBaseRelativePatcher::ReserveSpaceEnd(offset); 10471b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko} 10571b0ddf988208c9f77e7d2c1e75066cc3fe20c61Vladimir Marko 106b163bb742a099c1808907b513ae39068b63b1692Vladimir Markouint32_t Arm64RelativePatcher::WriteThunks(OutputStream* out, uint32_t offset) { 107b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (fix_cortex_a53_843419_) { 108b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (!current_method_thunks_.empty()) { 109b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t aligned_offset = CompiledMethod::AlignCode(offset, kArm64); 110b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (kIsDebugBuild) { 111b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko CHECK(IsAligned<kAdrpThunkSize>(current_method_thunks_.size())); 112b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko size_t num_thunks = current_method_thunks_.size() / kAdrpThunkSize; 113b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko CHECK_LE(num_thunks, processed_adrp_thunks_); 114b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko for (size_t i = 0u; i != num_thunks; ++i) { 115b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko const auto& entry = adrp_thunk_locations_[processed_adrp_thunks_ - num_thunks + i]; 116b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko CHECK_EQ(entry.second, aligned_offset + i * kAdrpThunkSize); 117b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 118b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 119b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t aligned_code_delta = aligned_offset - offset; 120b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (aligned_code_delta != 0u && !WriteCodeAlignment(out, aligned_code_delta)) { 121b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko return 0u; 122b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 123b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (!WriteMiscThunk(out, ArrayRef<const uint8_t>(current_method_thunks_))) { 124b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko return 0u; 125b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 126b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko offset = aligned_offset + current_method_thunks_.size(); 127b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko current_method_thunks_.clear(); 128b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 129b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 130b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko return ArmBaseRelativePatcher::WriteThunks(out, offset); 131b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko} 132b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 133b163bb742a099c1808907b513ae39068b63b1692Vladimir Markovoid Arm64RelativePatcher::PatchCall(std::vector<uint8_t>* code, uint32_t literal_offset, 134b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t patch_offset, uint32_t target_offset) { 135b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_LE(literal_offset + 4u, code->size()); 136b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_EQ(literal_offset & 3u, 0u); 137b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_EQ(patch_offset & 3u, 0u); 138b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_EQ(target_offset & 3u, 0u); 139b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t displacement = CalculateDisplacement(patch_offset, target_offset & ~1u); 140b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_EQ(displacement & 3u, 0u); 141b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK((displacement >> 27) == 0u || (displacement >> 27) == 31u); // 28-bit signed. 142b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t insn = (displacement & 0x0fffffffu) >> 2; 143b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko insn |= 0x94000000; // BL 144b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 145b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // Check that we're just overwriting an existing BL. 146b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_EQ(GetInsn(code, literal_offset) & 0xfc000000u, 0x94000000u); 147b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // Write the new BL. 148b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko SetInsn(code, literal_offset, insn); 149b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko} 150b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 151b163bb742a099c1808907b513ae39068b63b1692Vladimir Markovoid Arm64RelativePatcher::PatchDexCacheReference(std::vector<uint8_t>* code, 152b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko const LinkerPatch& patch, 153b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t patch_offset, 154b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t target_offset) { 155b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_EQ(patch_offset & 3u, 0u); 156b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_EQ(target_offset & 3u, 0u); 157b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t literal_offset = patch.LiteralOffset(); 158b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t insn = GetInsn(code, literal_offset); 159b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t pc_insn_offset = patch.PcInsnOffset(); 160b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t disp = target_offset - ((patch_offset - literal_offset + pc_insn_offset) & ~0xfffu); 1613d21bdf8894e780d349c481e5c9e29fe1556051cMathieu Chartier bool wide = (insn & 0x40000000) != 0; 1623d21bdf8894e780d349c481e5c9e29fe1556051cMathieu Chartier uint32_t shift = wide ? 3u : 2u; 163b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (literal_offset == pc_insn_offset) { 164b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // Check it's an ADRP with imm == 0 (unset). 165b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_EQ((insn & 0xffffffe0u), 0x90000000u) 166b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko << literal_offset << ", " << pc_insn_offset << ", 0x" << std::hex << insn; 167b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (fix_cortex_a53_843419_ && processed_adrp_thunks_ != adrp_thunk_locations_.size() && 168b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko adrp_thunk_locations_[processed_adrp_thunks_].first == patch_offset) { 169b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK(NeedsErratum843419Thunk(ArrayRef<const uint8_t>(*code), 170b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko literal_offset, patch_offset)); 171b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t thunk_offset = adrp_thunk_locations_[processed_adrp_thunks_].second; 172b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t adrp_disp = target_offset - (thunk_offset & ~0xfffu); 173b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t adrp = PatchAdrp(insn, adrp_disp); 174b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 175b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t out_disp = thunk_offset - patch_offset; 176b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_EQ(out_disp & 3u, 0u); 177b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK((out_disp >> 27) == 0u || (out_disp >> 27) == 31u); // 28-bit signed. 1783d21bdf8894e780d349c481e5c9e29fe1556051cMathieu Chartier insn = (out_disp & 0x0fffffffu) >> shift; 179b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko insn |= 0x14000000; // B <thunk> 180b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 181b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t back_disp = -out_disp; 182b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_EQ(back_disp & 3u, 0u); 183b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK((back_disp >> 27) == 0u || (back_disp >> 27) == 31u); // 28-bit signed. 184b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t b_back = (back_disp & 0x0fffffffu) >> 2; 185b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko b_back |= 0x14000000; // B <back> 186b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko size_t thunks_code_offset = current_method_thunks_.size(); 187b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko current_method_thunks_.resize(thunks_code_offset + kAdrpThunkSize); 188b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko SetInsn(¤t_method_thunks_, thunks_code_offset, adrp); 189b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko SetInsn(¤t_method_thunks_, thunks_code_offset + 4u, b_back); 190b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko static_assert(kAdrpThunkSize == 2 * 4u, "thunk has 2 instructions"); 191b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 192b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko processed_adrp_thunks_ += 1u; 193b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } else { 194b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko insn = PatchAdrp(insn, disp); 195b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 196b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // Write the new ADRP (or B to the erratum 843419 thunk). 197b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko SetInsn(code, literal_offset, insn); 198b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } else { 1993d21bdf8894e780d349c481e5c9e29fe1556051cMathieu Chartier // LDR 32-bit or 64-bit with imm12 == 0 (unset). 2003d21bdf8894e780d349c481e5c9e29fe1556051cMathieu Chartier DCHECK_EQ(insn & 0xbffffc00, 0xb9400000) << insn; 201b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (kIsDebugBuild) { 202b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t adrp = GetInsn(code, pc_insn_offset); 203b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if ((adrp & 0x9f000000u) != 0x90000000u) { 204b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko CHECK(fix_cortex_a53_843419_); 205b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko CHECK_EQ(adrp & 0xfc000000u, 0x14000000u); // B <thunk> 206b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko CHECK(IsAligned<kAdrpThunkSize>(current_method_thunks_.size())); 207b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko size_t num_thunks = current_method_thunks_.size() / kAdrpThunkSize; 208b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko CHECK_LE(num_thunks, processed_adrp_thunks_); 209b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t b_offset = patch_offset - literal_offset + pc_insn_offset; 210b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko for (size_t i = processed_adrp_thunks_ - num_thunks; ; ++i) { 211b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko CHECK_NE(i, processed_adrp_thunks_); 212b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if (adrp_thunk_locations_[i].first == b_offset) { 213b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko size_t idx = num_thunks - (processed_adrp_thunks_ - i); 214b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko adrp = GetInsn(¤t_method_thunks_, idx * kAdrpThunkSize); 215b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko break; 216b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 217b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 218b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 219b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko CHECK_EQ(adrp & 0x9f00001fu, // Check that pc_insn_offset points 220b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 0x90000000 | ((insn >> 5) & 0x1fu)); // to ADRP with matching register. 221b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 2223d21bdf8894e780d349c481e5c9e29fe1556051cMathieu Chartier uint32_t imm12 = (disp & 0xfffu) >> shift; 223b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko insn = (insn & ~(0xfffu << 10)) | (imm12 << 10); 224b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko SetInsn(code, literal_offset, insn); 225b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 226b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko} 227b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 228b163bb742a099c1808907b513ae39068b63b1692Vladimir Markostd::vector<uint8_t> Arm64RelativePatcher::CompileThunkCode() { 229b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // The thunk just uses the entry point in the ArtMethod. This works even for calls 230b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // to the generic JNI and interpreter trampolines. 231b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko arm64::Arm64Assembler assembler; 2323d21bdf8894e780d349c481e5c9e29fe1556051cMathieu Chartier Offset offset(ArtMethod::EntryPointFromQuickCompiledCodeOffset( 233b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko kArm64PointerSize).Int32Value()); 234b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko assembler.JumpTo(ManagedRegister(arm64::X0), offset, ManagedRegister(arm64::IP0)); 235b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // Ensure we emit the literal pool. 236b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko assembler.EmitSlowPaths(); 237b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko std::vector<uint8_t> thunk_code(assembler.CodeSize()); 238b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko MemoryRegion code(thunk_code.data(), thunk_code.size()); 239b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko assembler.FinalizeInstructions(code); 240b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko return thunk_code; 241b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko} 242b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 243b163bb742a099c1808907b513ae39068b63b1692Vladimir Markouint32_t Arm64RelativePatcher::PatchAdrp(uint32_t adrp, uint32_t disp) { 244b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko return (adrp & 0x9f00001fu) | // Clear offset bits, keep ADRP with destination reg. 245b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // Bottom 12 bits are ignored, the next 2 lowest bits are encoded in bits 29-30. 246b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko ((disp & 0x00003000u) << (29 - 12)) | 247b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // The next 16 bits are encoded in bits 5-22. 248b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko ((disp & 0xffffc000u) >> (12 + 2 - 5)) | 249b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // Since the target_offset is based on the beginning of the oat file and the 250b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // image space precedes the oat file, the target_offset into image space will 251b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // be negative yet passed as uint32_t. Therefore we limit the displacement 252b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // to +-2GiB (rather than the maximim +-4GiB) and determine the sign bit from 253b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko // the highest bit of the displacement. This is encoded in bit 23. 254b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko ((disp & 0x80000000u) >> (31 - 23)); 255b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko} 256b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 257b163bb742a099c1808907b513ae39068b63b1692Vladimir Markobool Arm64RelativePatcher::NeedsErratum843419Thunk(ArrayRef<const uint8_t> code, 258b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t literal_offset, 259b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t patch_offset) { 260b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_EQ(patch_offset & 0x3u, 0u); 261b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko if ((patch_offset & 0xff8) == 0xff8) { // ...ff8 or ...ffc 262b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t adrp = GetInsn(code, literal_offset); 263b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_EQ(adrp & 0xff000000, 0x90000000); 26497e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin uint32_t next_offset = patch_offset + 4u; 265b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint32_t next_insn = GetInsn(code, literal_offset + 4u); 26697e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin 26797e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin // Below we avoid patching sequences where the adrp is followed by a load which can easily 26897e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin // be proved to be aligned. 26997e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin 27097e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin // First check if the next insn is the LDR using the result of the ADRP. 27197e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin // LDR <Wt>, [<Xn>, #pimm], where <Xn> == ADRP destination reg. 27297e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin if ((next_insn & 0xffc00000) == 0xb9400000 && 27397e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin (((next_insn >> 5) ^ adrp) & 0x1f) == 0) { 27497e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin return false; 27597e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin } 27697e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin 27797e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin // LDR <Wt>, <label> is always aligned and thus it doesn't cause boundary crossing. 27897e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin if ((next_insn & 0xff000000) == 0x18000000) { 27997e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin return false; 28097e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin } 28197e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin 28297e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin // LDR <Xt>, <label> is aligned iff the pc + displacement is a multiple of 8. 28397e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin if ((next_insn & 0xff000000) == 0x58000000) { 28497e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin bool is_aligned_load = (((next_offset >> 2) ^ (next_insn >> 5)) & 1) == 0; 28597e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin return !is_aligned_load; 28697e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin } 28797e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin 28897e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin // LDR <Wt>, [SP, #<pimm>] and LDR <Xt>, [SP, #<pimm>] are always aligned loads, as SP is 28997e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin // guaranteed to be 128-bits aligned and <pimm> is multiple of the load size. 29097e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin if ((next_insn & 0xbfc003e0) == 0xb94003e0) { 29197e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin return false; 29297e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin } 29397e2f26524f4d08796a0a224b17e082734cceb5fMatteo Franchin return true; 294b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko } 295b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko return false; 296b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko} 297b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 298b163bb742a099c1808907b513ae39068b63b1692Vladimir Markovoid Arm64RelativePatcher::SetInsn(std::vector<uint8_t>* code, uint32_t offset, uint32_t value) { 299b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_LE(offset + 4u, code->size()); 300b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_EQ(offset & 3u, 0u); 301b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko uint8_t* addr = &(*code)[offset]; 302b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko addr[0] = (value >> 0) & 0xff; 303b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko addr[1] = (value >> 8) & 0xff; 304b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko addr[2] = (value >> 16) & 0xff; 305b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko addr[3] = (value >> 24) & 0xff; 306b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko} 307b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 308b163bb742a099c1808907b513ae39068b63b1692Vladimir Markouint32_t Arm64RelativePatcher::GetInsn(ArrayRef<const uint8_t> code, uint32_t offset) { 309b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_LE(offset + 4u, code.size()); 310b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko DCHECK_EQ(offset & 3u, 0u); 311b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko const uint8_t* addr = &code[offset]; 312b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko return 313b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko (static_cast<uint32_t>(addr[0]) << 0) + 314b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko (static_cast<uint32_t>(addr[1]) << 8) + 315b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko (static_cast<uint32_t>(addr[2]) << 16)+ 316b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko (static_cast<uint32_t>(addr[3]) << 24); 317b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko} 318b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 319b163bb742a099c1808907b513ae39068b63b1692Vladimir Markotemplate <typename Alloc> 320b163bb742a099c1808907b513ae39068b63b1692Vladimir Markouint32_t Arm64RelativePatcher::GetInsn(std::vector<uint8_t, Alloc>* code, uint32_t offset) { 321b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko return GetInsn(ArrayRef<const uint8_t>(*code), offset); 322b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko} 323b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko 324b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko} // namespace linker 325b163bb742a099c1808907b513ae39068b63b1692Vladimir Marko} // namespace art 326