1/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "linker/arm64/relative_patcher_arm64.h"
18
19#include "base/casts.h"
20#include "linker/relative_patcher_test.h"
21#include "lock_word.h"
22#include "mirror/array-inl.h"
23#include "mirror/object.h"
24#include "oat_quick_method_header.h"
25
26namespace art {
27namespace linker {
28
29class Arm64RelativePatcherTest : public RelativePatcherTest {
30 public:
31  explicit Arm64RelativePatcherTest(const std::string& variant)
32      : RelativePatcherTest(InstructionSet::kArm64, variant) { }
33
34 protected:
35  static const uint8_t kCallRawCode[];
36  static const ArrayRef<const uint8_t> kCallCode;
37  static const uint8_t kNopRawCode[];
38  static const ArrayRef<const uint8_t> kNopCode;
39
40  // NOP instruction.
41  static constexpr uint32_t kNopInsn = 0xd503201f;
42
43  // All branches can be created from kBlPlus0 or kBPlus0 by adding the low 26 bits.
44  static constexpr uint32_t kBlPlus0 = 0x94000000u;
45  static constexpr uint32_t kBPlus0 = 0x14000000u;
46
47  // Special BL values.
48  static constexpr uint32_t kBlPlusMax = 0x95ffffffu;
49  static constexpr uint32_t kBlMinusMax = 0x96000000u;
50
51  // LDR immediate, 32-bit, unsigned offset.
52  static constexpr uint32_t kLdrWInsn = 0xb9400000u;
53
54  // LDR register, 32-bit, LSL #2.
55  static constexpr uint32_t kLdrWLsl2Insn = 0xb8607800u;
56
57  // LDUR, 32-bit.
58  static constexpr uint32_t kLdurWInsn = 0xb8400000u;
59
60  // ADD/ADDS/SUB/SUBS immediate, 64-bit.
61  static constexpr uint32_t kAddXInsn = 0x91000000u;
62  static constexpr uint32_t kAddsXInsn = 0xb1000000u;
63  static constexpr uint32_t kSubXInsn = 0xd1000000u;
64  static constexpr uint32_t kSubsXInsn = 0xf1000000u;
65
66  // LDUR x2, [sp, #4], i.e. unaligned load crossing 64-bit boundary (assuming aligned sp).
67  static constexpr uint32_t kLdurInsn = 0xf840405fu;
68
69  // LDR w12, <label> and LDR x12, <label>. Bits 5-23 contain label displacement in 4-byte units.
70  static constexpr uint32_t kLdrWPcRelInsn = 0x1800000cu;
71  static constexpr uint32_t kLdrXPcRelInsn = 0x5800000cu;
72
73  // LDR w13, [SP, #<pimm>] and LDR x13, [SP, #<pimm>]. Bits 10-21 contain displacement from SP
74  // in units of 4-bytes (for 32-bit load) or 8-bytes (for 64-bit load).
75  static constexpr uint32_t kLdrWSpRelInsn = 0xb94003edu;
76  static constexpr uint32_t kLdrXSpRelInsn = 0xf94003edu;
77
78  // CBNZ x17, +0. Bits 5-23 are a placeholder for target offset from PC in units of 4-bytes.
79  static constexpr uint32_t kCbnzIP1Plus0Insn = 0xb5000011u;
80
81  void InsertInsn(std::vector<uint8_t>* code, size_t pos, uint32_t insn) {
82    CHECK_LE(pos, code->size());
83    const uint8_t insn_code[] = {
84        static_cast<uint8_t>(insn),
85        static_cast<uint8_t>(insn >> 8),
86        static_cast<uint8_t>(insn >> 16),
87        static_cast<uint8_t>(insn >> 24),
88    };
89    static_assert(sizeof(insn_code) == 4u, "Invalid sizeof(insn_code).");
90    code->insert(code->begin() + pos, insn_code, insn_code + sizeof(insn_code));
91  }
92
93  void PushBackInsn(std::vector<uint8_t>* code, uint32_t insn) {
94    InsertInsn(code, code->size(), insn);
95  }
96
97  std::vector<uint8_t> RawCode(std::initializer_list<uint32_t> insns) {
98    std::vector<uint8_t> raw_code;
99    raw_code.reserve(insns.size() * 4u);
100    for (uint32_t insn : insns) {
101      PushBackInsn(&raw_code, insn);
102    }
103    return raw_code;
104  }
105
106  uint32_t Create2MethodsWithGap(const ArrayRef<const uint8_t>& method1_code,
107                                 const ArrayRef<const LinkerPatch>& method1_patches,
108                                 const ArrayRef<const uint8_t>& last_method_code,
109                                 const ArrayRef<const LinkerPatch>& last_method_patches,
110                                 uint32_t distance_without_thunks) {
111    CHECK_EQ(distance_without_thunks % kArm64Alignment, 0u);
112    uint32_t method1_offset =
113        kTrampolineSize + CodeAlignmentSize(kTrampolineSize) + sizeof(OatQuickMethodHeader);
114    AddCompiledMethod(MethodRef(1u), method1_code, method1_patches);
115    const uint32_t gap_start = method1_offset + method1_code.size();
116
117    // We want to put the method3 at a very precise offset.
118    const uint32_t last_method_offset = method1_offset + distance_without_thunks;
119    CHECK_ALIGNED(last_method_offset, kArm64Alignment);
120    const uint32_t gap_end = last_method_offset - sizeof(OatQuickMethodHeader);
121
122    // Fill the gap with intermediate methods in chunks of 2MiB and the first in [2MiB, 4MiB).
123    // (This allows deduplicating the small chunks to avoid using 256MiB of memory for +-128MiB
124    // offsets by this test. Making the first chunk bigger makes it easy to give all intermediate
125    // methods the same alignment of the end, so the thunk insertion adds a predictable size as
126    // long as it's after the first chunk.)
127    uint32_t method_idx = 2u;
128    constexpr uint32_t kSmallChunkSize = 2 * MB;
129    std::vector<uint8_t> gap_code;
130    uint32_t gap_size = gap_end - gap_start;
131    uint32_t num_small_chunks = std::max(gap_size / kSmallChunkSize, 1u) - 1u;
132    uint32_t chunk_start = gap_start;
133    uint32_t chunk_size = gap_size - num_small_chunks * kSmallChunkSize;
134    for (uint32_t i = 0; i <= num_small_chunks; ++i) {  // num_small_chunks+1 iterations.
135      uint32_t chunk_code_size =
136          chunk_size - CodeAlignmentSize(chunk_start) - sizeof(OatQuickMethodHeader);
137      gap_code.resize(chunk_code_size, 0u);
138      AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(gap_code));
139      method_idx += 1u;
140      chunk_start += chunk_size;
141      chunk_size = kSmallChunkSize;  // For all but the first chunk.
142      DCHECK_EQ(CodeAlignmentSize(gap_end), CodeAlignmentSize(chunk_start));
143    }
144
145    // Add the last method and link
146    AddCompiledMethod(MethodRef(method_idx), last_method_code, last_method_patches);
147    Link();
148
149    // Check assumptions.
150    CHECK_EQ(GetMethodOffset(1), method1_offset);
151    auto last_result = method_offset_map_.FindMethodOffset(MethodRef(method_idx));
152    CHECK(last_result.first);
153    // There may be a thunk before method2.
154    if (last_result.second != last_method_offset) {
155      // Thunk present. Check that there's only one.
156      uint32_t thunk_end =
157          CompiledCode::AlignCode(gap_end, InstructionSet::kArm64) + MethodCallThunkSize();
158      uint32_t header_offset = thunk_end + CodeAlignmentSize(thunk_end);
159      CHECK_EQ(last_result.second, header_offset + sizeof(OatQuickMethodHeader));
160    }
161    return method_idx;
162  }
163
164  uint32_t GetMethodOffset(uint32_t method_idx) {
165    auto result = method_offset_map_.FindMethodOffset(MethodRef(method_idx));
166    CHECK(result.first);
167    CHECK_ALIGNED(result.second, 4u);
168    return result.second;
169  }
170
171  std::vector<uint8_t> CompileMethodCallThunk() {
172    ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetMethodCallKey();
173    return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
174  }
175
176  uint32_t MethodCallThunkSize() {
177    return CompileMethodCallThunk().size();
178  }
179
180  bool CheckThunk(uint32_t thunk_offset) {
181    const std::vector<uint8_t> expected_code = CompileMethodCallThunk();
182    if (output_.size() < thunk_offset + expected_code.size()) {
183      LOG(ERROR) << "output_.size() == " << output_.size() << " < "
184          << "thunk_offset + expected_code.size() == " << (thunk_offset + expected_code.size());
185      return false;
186    }
187    ArrayRef<const uint8_t> linked_code(&output_[thunk_offset], expected_code.size());
188    if (linked_code == ArrayRef<const uint8_t>(expected_code)) {
189      return true;
190    }
191    // Log failure info.
192    DumpDiff(ArrayRef<const uint8_t>(expected_code), linked_code);
193    return false;
194  }
195
196  std::vector<uint8_t> GenNops(size_t num_nops) {
197    std::vector<uint8_t> result;
198    result.reserve(num_nops * 4u);
199    for (size_t i = 0; i != num_nops; ++i) {
200      PushBackInsn(&result, kNopInsn);
201    }
202    return result;
203  }
204
205  std::vector<uint8_t> GenNopsAndBl(size_t num_nops, uint32_t bl) {
206    std::vector<uint8_t> result;
207    result.reserve(num_nops * 4u + 4u);
208    for (size_t i = 0; i != num_nops; ++i) {
209      PushBackInsn(&result, kNopInsn);
210    }
211    PushBackInsn(&result, bl);
212    return result;
213  }
214
215  std::vector<uint8_t> GenNopsAndAdrpAndUse(size_t num_nops,
216                                            uint32_t method_offset,
217                                            uint32_t target_offset,
218                                            uint32_t use_insn) {
219    std::vector<uint8_t> result;
220    result.reserve(num_nops * 4u + 8u);
221    for (size_t i = 0; i != num_nops; ++i) {
222      PushBackInsn(&result, kNopInsn);
223    }
224    CHECK_ALIGNED(method_offset, 4u);
225    CHECK_ALIGNED(target_offset, 4u);
226    uint32_t adrp_offset = method_offset + num_nops * 4u;
227    uint32_t disp = target_offset - (adrp_offset & ~0xfffu);
228    if (use_insn == kLdrWInsn) {
229      DCHECK_ALIGNED(disp, 1u << 2);
230      use_insn |= 1 |                         // LDR x1, [x0, #(imm12 << 2)]
231          ((disp & 0xfffu) << (10 - 2));      // imm12 = ((disp & 0xfffu) >> 2) is at bit 10.
232    } else if (use_insn == kAddXInsn) {
233      use_insn |= 1 |                         // ADD x1, x0, #imm
234          (disp & 0xfffu) << 10;              // imm12 = (disp & 0xfffu) is at bit 10.
235    } else {
236      LOG(FATAL) << "Unexpected instruction: 0x" << std::hex << use_insn;
237    }
238    uint32_t adrp = 0x90000000u |             // ADRP x0, +SignExtend(immhi:immlo:Zeros(12), 64)
239        ((disp & 0x3000u) << (29 - 12)) |     // immlo = ((disp & 0x3000u) >> 12) is at bit 29,
240        ((disp & 0xffffc000) >> (14 - 5)) |   // immhi = (disp >> 14) is at bit 5,
241        // We take the sign bit from the disp, limiting disp to +- 2GiB.
242        ((disp & 0x80000000) >> (31 - 23));   // sign bit in immhi is at bit 23.
243    PushBackInsn(&result, adrp);
244    PushBackInsn(&result, use_insn);
245    return result;
246  }
247
248  std::vector<uint8_t> GenNopsAndAdrpLdr(size_t num_nops,
249                                         uint32_t method_offset,
250                                         uint32_t target_offset) {
251    return GenNopsAndAdrpAndUse(num_nops, method_offset, target_offset, kLdrWInsn);
252  }
253
254  void TestNopsAdrpLdr(size_t num_nops, uint32_t bss_begin, uint32_t string_entry_offset) {
255    constexpr uint32_t kStringIndex = 1u;
256    string_index_to_offset_map_.Put(kStringIndex, string_entry_offset);
257    bss_begin_ = bss_begin;
258    auto code = GenNopsAndAdrpLdr(num_nops, 0u, 0u);  // Unpatched.
259    const LinkerPatch patches[] = {
260        LinkerPatch::StringBssEntryPatch(num_nops * 4u     , nullptr, num_nops * 4u, kStringIndex),
261        LinkerPatch::StringBssEntryPatch(num_nops * 4u + 4u, nullptr, num_nops * 4u, kStringIndex),
262    };
263    AddCompiledMethod(MethodRef(1u),
264                      ArrayRef<const uint8_t>(code),
265                      ArrayRef<const LinkerPatch>(patches));
266    Link();
267
268    uint32_t method1_offset = GetMethodOffset(1u);
269    uint32_t target_offset = bss_begin_ + string_entry_offset;
270    auto expected_code = GenNopsAndAdrpLdr(num_nops, method1_offset, target_offset);
271    EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
272  }
273
274  std::vector<uint8_t> GenNopsAndAdrpAdd(size_t num_nops,
275                                         uint32_t method_offset,
276                                         uint32_t target_offset) {
277    return GenNopsAndAdrpAndUse(num_nops, method_offset, target_offset, kAddXInsn);
278  }
279
280  void TestNopsAdrpAdd(size_t num_nops, uint32_t string_offset) {
281    constexpr uint32_t kStringIndex = 1u;
282    string_index_to_offset_map_.Put(kStringIndex, string_offset);
283    auto code = GenNopsAndAdrpAdd(num_nops, 0u, 0u);  // Unpatched.
284    const LinkerPatch patches[] = {
285        LinkerPatch::RelativeStringPatch(num_nops * 4u     , nullptr, num_nops * 4u, kStringIndex),
286        LinkerPatch::RelativeStringPatch(num_nops * 4u + 4u, nullptr, num_nops * 4u, kStringIndex),
287    };
288    AddCompiledMethod(MethodRef(1u),
289                      ArrayRef<const uint8_t>(code),
290                      ArrayRef<const LinkerPatch>(patches));
291    Link();
292
293    uint32_t method1_offset = GetMethodOffset(1u);
294    auto expected_code = GenNopsAndAdrpAdd(num_nops, method1_offset, string_offset);
295    EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
296  }
297
298  void PrepareNopsAdrpInsn2Ldr(size_t num_nops,
299                               uint32_t insn2,
300                               uint32_t bss_begin,
301                               uint32_t string_entry_offset) {
302    constexpr uint32_t kStringIndex = 1u;
303    string_index_to_offset_map_.Put(kStringIndex, string_entry_offset);
304    bss_begin_ = bss_begin;
305    auto code = GenNopsAndAdrpLdr(num_nops, 0u, 0u);  // Unpatched.
306    InsertInsn(&code, num_nops * 4u + 4u, insn2);
307    const LinkerPatch patches[] = {
308        LinkerPatch::StringBssEntryPatch(num_nops * 4u     , nullptr, num_nops * 4u, kStringIndex),
309        LinkerPatch::StringBssEntryPatch(num_nops * 4u + 8u, nullptr, num_nops * 4u, kStringIndex),
310    };
311    AddCompiledMethod(MethodRef(1u),
312                      ArrayRef<const uint8_t>(code),
313                      ArrayRef<const LinkerPatch>(patches));
314    Link();
315  }
316
317  void PrepareNopsAdrpInsn2Add(size_t num_nops, uint32_t insn2, uint32_t string_offset) {
318    constexpr uint32_t kStringIndex = 1u;
319    string_index_to_offset_map_.Put(kStringIndex, string_offset);
320    auto code = GenNopsAndAdrpAdd(num_nops, 0u, 0u);  // Unpatched.
321    InsertInsn(&code, num_nops * 4u + 4u, insn2);
322    const LinkerPatch patches[] = {
323        LinkerPatch::RelativeStringPatch(num_nops * 4u     , nullptr, num_nops * 4u, kStringIndex),
324        LinkerPatch::RelativeStringPatch(num_nops * 4u + 8u, nullptr, num_nops * 4u, kStringIndex),
325    };
326    AddCompiledMethod(MethodRef(1u),
327                      ArrayRef<const uint8_t>(code),
328                      ArrayRef<const LinkerPatch>(patches));
329    Link();
330  }
331
332  void TestNopsAdrpInsn2AndUse(size_t num_nops,
333                               uint32_t insn2,
334                               uint32_t target_offset,
335                               uint32_t use_insn) {
336    uint32_t method1_offset = GetMethodOffset(1u);
337    auto expected_code = GenNopsAndAdrpAndUse(num_nops, method1_offset, target_offset, use_insn);
338    InsertInsn(&expected_code, num_nops * 4u + 4u, insn2);
339    EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
340  }
341
342  void TestNopsAdrpInsn2AndUseHasThunk(size_t num_nops,
343                                       uint32_t insn2,
344                                       uint32_t target_offset,
345                                       uint32_t use_insn) {
346    uint32_t method1_offset = GetMethodOffset(1u);
347    CHECK(!compiled_method_refs_.empty());
348    CHECK_EQ(compiled_method_refs_[0].index, 1u);
349    CHECK_EQ(compiled_method_refs_.size(), compiled_methods_.size());
350    uint32_t method1_size = compiled_methods_[0]->GetQuickCode().size();
351    uint32_t thunk_offset =
352        CompiledCode::AlignCode(method1_offset + method1_size, InstructionSet::kArm64);
353    uint32_t b_diff = thunk_offset - (method1_offset + num_nops * 4u);
354    CHECK_ALIGNED(b_diff, 4u);
355    ASSERT_LT(b_diff, 128 * MB);
356    uint32_t b_out = kBPlus0 + ((b_diff >> 2) & 0x03ffffffu);
357    uint32_t b_in = kBPlus0 + ((-b_diff >> 2) & 0x03ffffffu);
358
359    auto expected_code = GenNopsAndAdrpAndUse(num_nops, method1_offset, target_offset, use_insn);
360    InsertInsn(&expected_code, num_nops * 4u + 4u, insn2);
361    // Replace adrp with bl.
362    expected_code.erase(expected_code.begin() + num_nops * 4u,
363                        expected_code.begin() + num_nops * 4u + 4u);
364    InsertInsn(&expected_code, num_nops * 4u, b_out);
365    EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
366
367    auto expected_thunk_code = GenNopsAndAdrpLdr(0u, thunk_offset, target_offset);
368    ASSERT_EQ(expected_thunk_code.size(), 8u);
369    expected_thunk_code.erase(expected_thunk_code.begin() + 4u, expected_thunk_code.begin() + 8u);
370    InsertInsn(&expected_thunk_code, 4u, b_in);
371    ASSERT_EQ(expected_thunk_code.size(), 8u);
372
373    uint32_t thunk_size = MethodCallThunkSize();
374    ASSERT_EQ(thunk_offset + thunk_size, output_.size());
375    ASSERT_EQ(thunk_size, expected_thunk_code.size());
376    ArrayRef<const uint8_t> thunk_code(&output_[thunk_offset], thunk_size);
377    if (ArrayRef<const uint8_t>(expected_thunk_code) != thunk_code) {
378      DumpDiff(ArrayRef<const uint8_t>(expected_thunk_code), thunk_code);
379      FAIL();
380    }
381  }
382
383  void TestAdrpInsn2Ldr(uint32_t insn2,
384                        uint32_t adrp_offset,
385                        bool has_thunk,
386                        uint32_t bss_begin,
387                        uint32_t string_entry_offset) {
388    uint32_t method1_offset =
389        kTrampolineSize + CodeAlignmentSize(kTrampolineSize) + sizeof(OatQuickMethodHeader);
390    ASSERT_LT(method1_offset, adrp_offset);
391    CHECK_ALIGNED(adrp_offset, 4u);
392    uint32_t num_nops = (adrp_offset - method1_offset) / 4u;
393    PrepareNopsAdrpInsn2Ldr(num_nops, insn2, bss_begin, string_entry_offset);
394    uint32_t target_offset = bss_begin_ + string_entry_offset;
395    if (has_thunk) {
396      TestNopsAdrpInsn2AndUseHasThunk(num_nops, insn2, target_offset, kLdrWInsn);
397    } else {
398      TestNopsAdrpInsn2AndUse(num_nops, insn2, target_offset, kLdrWInsn);
399    }
400    ASSERT_EQ(method1_offset, GetMethodOffset(1u));  // If this fails, num_nops is wrong.
401  }
402
403  void TestAdrpLdurLdr(uint32_t adrp_offset,
404                       bool has_thunk,
405                       uint32_t bss_begin,
406                       uint32_t string_entry_offset) {
407    TestAdrpInsn2Ldr(kLdurInsn, adrp_offset, has_thunk, bss_begin, string_entry_offset);
408  }
409
410  void TestAdrpLdrPcRelLdr(uint32_t pcrel_ldr_insn,
411                           int32_t pcrel_disp,
412                           uint32_t adrp_offset,
413                           bool has_thunk,
414                           uint32_t bss_begin,
415                           uint32_t string_entry_offset) {
416    ASSERT_LT(pcrel_disp, 0x100000);
417    ASSERT_GE(pcrel_disp, -0x100000);
418    ASSERT_EQ(pcrel_disp & 0x3, 0);
419    uint32_t insn2 = pcrel_ldr_insn | (((static_cast<uint32_t>(pcrel_disp) >> 2) & 0x7ffffu) << 5);
420    TestAdrpInsn2Ldr(insn2, adrp_offset, has_thunk, bss_begin, string_entry_offset);
421  }
422
423  void TestAdrpLdrSpRelLdr(uint32_t sprel_ldr_insn,
424                           uint32_t sprel_disp_in_load_units,
425                           uint32_t adrp_offset,
426                           bool has_thunk,
427                           uint32_t bss_begin,
428                           uint32_t string_entry_offset) {
429    ASSERT_LT(sprel_disp_in_load_units, 0x1000u);
430    uint32_t insn2 = sprel_ldr_insn | ((sprel_disp_in_load_units & 0xfffu) << 10);
431    TestAdrpInsn2Ldr(insn2, adrp_offset, has_thunk, bss_begin, string_entry_offset);
432  }
433
434  void TestAdrpInsn2Add(uint32_t insn2,
435                        uint32_t adrp_offset,
436                        bool has_thunk,
437                        uint32_t string_offset) {
438    uint32_t method1_offset =
439        kTrampolineSize + CodeAlignmentSize(kTrampolineSize) + sizeof(OatQuickMethodHeader);
440    ASSERT_LT(method1_offset, adrp_offset);
441    CHECK_ALIGNED(adrp_offset, 4u);
442    uint32_t num_nops = (adrp_offset - method1_offset) / 4u;
443    PrepareNopsAdrpInsn2Add(num_nops, insn2, string_offset);
444    if (has_thunk) {
445      TestNopsAdrpInsn2AndUseHasThunk(num_nops, insn2, string_offset, kAddXInsn);
446    } else {
447      TestNopsAdrpInsn2AndUse(num_nops, insn2, string_offset, kAddXInsn);
448    }
449    ASSERT_EQ(method1_offset, GetMethodOffset(1u));  // If this fails, num_nops is wrong.
450  }
451
452  void TestAdrpLdurAdd(uint32_t adrp_offset, bool has_thunk, uint32_t string_offset) {
453    TestAdrpInsn2Add(kLdurInsn, adrp_offset, has_thunk, string_offset);
454  }
455
456  void TestAdrpLdrPcRelAdd(uint32_t pcrel_ldr_insn,
457                           int32_t pcrel_disp,
458                           uint32_t adrp_offset,
459                           bool has_thunk,
460                           uint32_t string_offset) {
461    ASSERT_LT(pcrel_disp, 0x100000);
462    ASSERT_GE(pcrel_disp, -0x100000);
463    ASSERT_EQ(pcrel_disp & 0x3, 0);
464    uint32_t insn2 = pcrel_ldr_insn | (((static_cast<uint32_t>(pcrel_disp) >> 2) & 0x7ffffu) << 5);
465    TestAdrpInsn2Add(insn2, adrp_offset, has_thunk, string_offset);
466  }
467
468  void TestAdrpLdrSpRelAdd(uint32_t sprel_ldr_insn,
469                           uint32_t sprel_disp_in_load_units,
470                           uint32_t adrp_offset,
471                           bool has_thunk,
472                           uint32_t string_offset) {
473    ASSERT_LT(sprel_disp_in_load_units, 0x1000u);
474    uint32_t insn2 = sprel_ldr_insn | ((sprel_disp_in_load_units & 0xfffu) << 10);
475    TestAdrpInsn2Add(insn2, adrp_offset, has_thunk, string_offset);
476  }
477
478  std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg, uint32_t holder_reg) {
479    const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
480        0u, Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg));
481    ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
482    return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
483  }
484
485  std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) {
486    LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
487        0u, Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg));
488    ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
489    return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
490  }
491
492  std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg) {
493    LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
494        0u, Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg));
495    ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
496    return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
497  }
498
499  uint32_t GetOutputInsn(uint32_t offset) {
500    CHECK_LE(offset, output_.size());
501    CHECK_GE(output_.size() - offset, 4u);
502    return (static_cast<uint32_t>(output_[offset]) << 0) |
503           (static_cast<uint32_t>(output_[offset + 1]) << 8) |
504           (static_cast<uint32_t>(output_[offset + 2]) << 16) |
505           (static_cast<uint32_t>(output_[offset + 3]) << 24);
506  }
507
508  void TestBakerField(uint32_t offset, uint32_t ref_reg);
509};
510
511const uint8_t Arm64RelativePatcherTest::kCallRawCode[] = {
512    0x00, 0x00, 0x00, 0x94
513};
514
515const ArrayRef<const uint8_t> Arm64RelativePatcherTest::kCallCode(kCallRawCode);
516
517const uint8_t Arm64RelativePatcherTest::kNopRawCode[] = {
518    0x1f, 0x20, 0x03, 0xd5
519};
520
521const ArrayRef<const uint8_t> Arm64RelativePatcherTest::kNopCode(kNopRawCode);
522
523class Arm64RelativePatcherTestDefault : public Arm64RelativePatcherTest {
524 public:
525  Arm64RelativePatcherTestDefault() : Arm64RelativePatcherTest("default") { }
526};
527
528class Arm64RelativePatcherTestDenver64 : public Arm64RelativePatcherTest {
529 public:
530  Arm64RelativePatcherTestDenver64() : Arm64RelativePatcherTest("denver64") { }
531};
532
533TEST_F(Arm64RelativePatcherTestDefault, CallSelf) {
534  const LinkerPatch patches[] = {
535      LinkerPatch::RelativeCodePatch(0u, nullptr, 1u),
536  };
537  AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
538  Link();
539
540  const std::vector<uint8_t> expected_code = RawCode({kBlPlus0});
541  EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
542}
543
544TEST_F(Arm64RelativePatcherTestDefault, CallOther) {
545  const LinkerPatch method1_patches[] = {
546      LinkerPatch::RelativeCodePatch(0u, nullptr, 2u),
547  };
548  AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(method1_patches));
549  const LinkerPatch method2_patches[] = {
550      LinkerPatch::RelativeCodePatch(0u, nullptr, 1u),
551  };
552  AddCompiledMethod(MethodRef(2u), kCallCode, ArrayRef<const LinkerPatch>(method2_patches));
553  Link();
554
555  uint32_t method1_offset = GetMethodOffset(1u);
556  uint32_t method2_offset = GetMethodOffset(2u);
557  uint32_t diff_after = method2_offset - method1_offset;
558  CHECK_ALIGNED(diff_after, 4u);
559  ASSERT_LT(diff_after >> 2, 1u << 8);  // Simple encoding, (diff_after >> 2) fits into 8 bits.
560  const std::vector<uint8_t> method1_expected_code = RawCode({kBlPlus0 + (diff_after >> 2)});
561  EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(method1_expected_code)));
562  uint32_t diff_before = method1_offset - method2_offset;
563  CHECK_ALIGNED(diff_before, 4u);
564  ASSERT_GE(diff_before, -1u << 27);
565  auto method2_expected_code = GenNopsAndBl(0u, kBlPlus0 | ((diff_before >> 2) & 0x03ffffffu));
566  EXPECT_TRUE(CheckLinkedMethod(MethodRef(2u), ArrayRef<const uint8_t>(method2_expected_code)));
567}
568
569TEST_F(Arm64RelativePatcherTestDefault, CallTrampoline) {
570  const LinkerPatch patches[] = {
571      LinkerPatch::RelativeCodePatch(0u, nullptr, 2u),
572  };
573  AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
574  Link();
575
576  uint32_t method1_offset = GetMethodOffset(1u);
577  uint32_t diff = kTrampolineOffset - method1_offset;
578  ASSERT_EQ(diff & 1u, 0u);
579  ASSERT_GE(diff, -1u << 9);  // Simple encoding, -256 <= (diff >> 1) < 0 (checked as unsigned).
580  auto expected_code = GenNopsAndBl(0u, kBlPlus0 | ((diff >> 2) & 0x03ffffffu));
581  EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
582}
583
584TEST_F(Arm64RelativePatcherTestDefault, CallTrampolineTooFar) {
585  constexpr uint32_t missing_method_index = 1024u;
586  auto last_method_raw_code = GenNopsAndBl(1u, kBlPlus0);
587  constexpr uint32_t bl_offset_in_last_method = 1u * 4u;  // After NOPs.
588  ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
589  ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
590  const LinkerPatch last_method_patches[] = {
591      LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, missing_method_index),
592  };
593
594  constexpr uint32_t just_over_max_negative_disp = 128 * MB + 4;
595  uint32_t last_method_idx = Create2MethodsWithGap(
596      kNopCode, ArrayRef<const LinkerPatch>(), last_method_code,
597      ArrayRef<const LinkerPatch>(last_method_patches),
598      just_over_max_negative_disp - bl_offset_in_last_method);
599  uint32_t method1_offset = GetMethodOffset(1u);
600  uint32_t last_method_offset = GetMethodOffset(last_method_idx);
601  ASSERT_EQ(method1_offset,
602            last_method_offset + bl_offset_in_last_method - just_over_max_negative_disp);
603  ASSERT_FALSE(method_offset_map_.FindMethodOffset(MethodRef(missing_method_index)).first);
604
605  // Check linked code.
606  uint32_t thunk_offset =
607      CompiledCode::AlignCode(last_method_offset + last_method_code.size(), InstructionSet::kArm64);
608  uint32_t diff = thunk_offset - (last_method_offset + bl_offset_in_last_method);
609  CHECK_ALIGNED(diff, 4u);
610  ASSERT_LT(diff, 128 * MB);
611  auto expected_code = GenNopsAndBl(1u, kBlPlus0 | (diff >> 2));
612  EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
613                                ArrayRef<const uint8_t>(expected_code)));
614  EXPECT_TRUE(CheckThunk(thunk_offset));
615}
616
617TEST_F(Arm64RelativePatcherTestDefault, CallOtherAlmostTooFarAfter) {
618  auto method1_raw_code = GenNopsAndBl(1u, kBlPlus0);
619  constexpr uint32_t bl_offset_in_method1 = 1u * 4u;  // After NOPs.
620  ArrayRef<const uint8_t> method1_code(method1_raw_code);
621  ASSERT_EQ(bl_offset_in_method1 + 4u, method1_code.size());
622  uint32_t expected_last_method_idx = 65;  // Based on 2MiB chunks in Create2MethodsWithGap().
623  const LinkerPatch method1_patches[] = {
624      LinkerPatch::RelativeCodePatch(bl_offset_in_method1, nullptr, expected_last_method_idx),
625  };
626
627  constexpr uint32_t max_positive_disp = 128 * MB - 4u;
628  uint32_t last_method_idx = Create2MethodsWithGap(method1_code,
629                                                   ArrayRef<const LinkerPatch>(method1_patches),
630                                                   kNopCode,
631                                                   ArrayRef<const LinkerPatch>(),
632                                                   bl_offset_in_method1 + max_positive_disp);
633  ASSERT_EQ(expected_last_method_idx, last_method_idx);
634
635  uint32_t method1_offset = GetMethodOffset(1u);
636  uint32_t last_method_offset = GetMethodOffset(last_method_idx);
637  ASSERT_EQ(method1_offset + bl_offset_in_method1 + max_positive_disp, last_method_offset);
638
639  // Check linked code.
640  auto expected_code = GenNopsAndBl(1u, kBlPlusMax);
641  EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
642}
643
644TEST_F(Arm64RelativePatcherTestDefault, CallOtherAlmostTooFarBefore) {
645  auto last_method_raw_code = GenNopsAndBl(0u, kBlPlus0);
646  constexpr uint32_t bl_offset_in_last_method = 0u * 4u;  // After NOPs.
647  ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
648  ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
649  const LinkerPatch last_method_patches[] = {
650      LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, 1u),
651  };
652
653  constexpr uint32_t max_negative_disp = 128 * MB;
654  uint32_t last_method_idx = Create2MethodsWithGap(kNopCode,
655                                                   ArrayRef<const LinkerPatch>(),
656                                                   last_method_code,
657                                                   ArrayRef<const LinkerPatch>(last_method_patches),
658                                                   max_negative_disp - bl_offset_in_last_method);
659  uint32_t method1_offset = GetMethodOffset(1u);
660  uint32_t last_method_offset = GetMethodOffset(last_method_idx);
661  ASSERT_EQ(method1_offset, last_method_offset + bl_offset_in_last_method - max_negative_disp);
662
663  // Check linked code.
664  auto expected_code = GenNopsAndBl(0u, kBlMinusMax);
665  EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
666                                ArrayRef<const uint8_t>(expected_code)));
667}
668
669TEST_F(Arm64RelativePatcherTestDefault, CallOtherJustTooFarAfter) {
670  auto method1_raw_code = GenNopsAndBl(0u, kBlPlus0);
671  constexpr uint32_t bl_offset_in_method1 = 0u * 4u;  // After NOPs.
672  ArrayRef<const uint8_t> method1_code(method1_raw_code);
673  ASSERT_EQ(bl_offset_in_method1 + 4u, method1_code.size());
674  uint32_t expected_last_method_idx = 65;  // Based on 2MiB chunks in Create2MethodsWithGap().
675  const LinkerPatch method1_patches[] = {
676      LinkerPatch::RelativeCodePatch(bl_offset_in_method1, nullptr, expected_last_method_idx),
677  };
678
679  constexpr uint32_t just_over_max_positive_disp = 128 * MB;
680  uint32_t last_method_idx = Create2MethodsWithGap(
681      method1_code,
682      ArrayRef<const LinkerPatch>(method1_patches),
683      kNopCode,
684      ArrayRef<const LinkerPatch>(),
685      bl_offset_in_method1 + just_over_max_positive_disp);
686  ASSERT_EQ(expected_last_method_idx, last_method_idx);
687
688  uint32_t method1_offset = GetMethodOffset(1u);
689  uint32_t last_method_offset = GetMethodOffset(last_method_idx);
690  ASSERT_TRUE(IsAligned<kArm64Alignment>(last_method_offset));
691  uint32_t last_method_header_offset = last_method_offset - sizeof(OatQuickMethodHeader);
692  uint32_t thunk_size = MethodCallThunkSize();
693  uint32_t thunk_offset = RoundDown(last_method_header_offset - thunk_size, kArm64Alignment);
694  DCHECK_EQ(thunk_offset + thunk_size + CodeAlignmentSize(thunk_offset + thunk_size),
695            last_method_header_offset);
696  uint32_t diff = thunk_offset - (method1_offset + bl_offset_in_method1);
697  CHECK_ALIGNED(diff, 4u);
698  ASSERT_LT(diff, 128 * MB);
699  auto expected_code = GenNopsAndBl(0u, kBlPlus0 | (diff >> 2));
700  EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
701  CheckThunk(thunk_offset);
702}
703
704TEST_F(Arm64RelativePatcherTestDefault, CallOtherJustTooFarBefore) {
705  auto last_method_raw_code = GenNopsAndBl(1u, kBlPlus0);
706  constexpr uint32_t bl_offset_in_last_method = 1u * 4u;  // After NOPs.
707  ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
708  ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
709  const LinkerPatch last_method_patches[] = {
710      LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, 1u),
711  };
712
713  constexpr uint32_t just_over_max_negative_disp = 128 * MB + 4;
714  uint32_t last_method_idx = Create2MethodsWithGap(
715      kNopCode, ArrayRef<const LinkerPatch>(), last_method_code,
716      ArrayRef<const LinkerPatch>(last_method_patches),
717      just_over_max_negative_disp - bl_offset_in_last_method);
718  uint32_t method1_offset = GetMethodOffset(1u);
719  uint32_t last_method_offset = GetMethodOffset(last_method_idx);
720  ASSERT_EQ(method1_offset,
721            last_method_offset + bl_offset_in_last_method - just_over_max_negative_disp);
722
723  // Check linked code.
724  uint32_t thunk_offset =
725      CompiledCode::AlignCode(last_method_offset + last_method_code.size(), InstructionSet::kArm64);
726  uint32_t diff = thunk_offset - (last_method_offset + bl_offset_in_last_method);
727  CHECK_ALIGNED(diff, 4u);
728  ASSERT_LT(diff, 128 * MB);
729  auto expected_code = GenNopsAndBl(1u, kBlPlus0 | (diff >> 2));
730  EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
731                                ArrayRef<const uint8_t>(expected_code)));
732  EXPECT_TRUE(CheckThunk(thunk_offset));
733}
734
735TEST_F(Arm64RelativePatcherTestDefault, StringBssEntry1) {
736  TestNopsAdrpLdr(0u, 0x12345678u, 0x1234u);
737}
738
739TEST_F(Arm64RelativePatcherTestDefault, StringBssEntry2) {
740  TestNopsAdrpLdr(0u, -0x12345678u, 0x4444u);
741}
742
743TEST_F(Arm64RelativePatcherTestDefault, StringBssEntry3) {
744  TestNopsAdrpLdr(0u, 0x12345000u, 0x3ffcu);
745}
746
747TEST_F(Arm64RelativePatcherTestDefault, StringBssEntry4) {
748  TestNopsAdrpLdr(0u, 0x12345000u, 0x4000u);
749}
750
751TEST_F(Arm64RelativePatcherTestDefault, StringReference1) {
752  TestNopsAdrpAdd(0u, 0x12345678u);
753}
754
755TEST_F(Arm64RelativePatcherTestDefault, StringReference2) {
756  TestNopsAdrpAdd(0u, -0x12345678u);
757}
758
759TEST_F(Arm64RelativePatcherTestDefault, StringReference3) {
760  TestNopsAdrpAdd(0u, 0x12345000u);
761}
762
763TEST_F(Arm64RelativePatcherTestDefault, StringReference4) {
764  TestNopsAdrpAdd(0u, 0x12345ffcu);
765}
766
767#define TEST_FOR_OFFSETS(test, disp1, disp2) \
768  test(0xff4u, disp1) test(0xff8u, disp1) test(0xffcu, disp1) test(0x1000u, disp1) \
769  test(0xff4u, disp2) test(0xff8u, disp2) test(0xffcu, disp2) test(0x1000u, disp2)
770
771#define DEFAULT_LDUR_LDR_TEST(adrp_offset, disp) \
772  TEST_F(Arm64RelativePatcherTestDefault, StringBssEntry ## adrp_offset ## Ldur ## disp) { \
773    bool has_thunk = ((adrp_offset) == 0xff8u || (adrp_offset) == 0xffcu); \
774    TestAdrpLdurLdr(adrp_offset, has_thunk, 0x12345678u, disp); \
775  }
776
777TEST_FOR_OFFSETS(DEFAULT_LDUR_LDR_TEST, 0x1234, 0x1238)
778
779#define DENVER64_LDUR_LDR_TEST(adrp_offset, disp) \
780  TEST_F(Arm64RelativePatcherTestDenver64, StringBssEntry ## adrp_offset ## Ldur ## disp) { \
781    TestAdrpLdurLdr(adrp_offset, false, 0x12345678u, disp); \
782  }
783
784TEST_FOR_OFFSETS(DENVER64_LDUR_LDR_TEST, 0x1234, 0x1238)
785
786// LDR <Wt>, <label> is always aligned. We should never have to use a fixup.
787#define LDRW_PCREL_LDR_TEST(adrp_offset, disp) \
788  TEST_F(Arm64RelativePatcherTestDefault, StringBssEntry ## adrp_offset ## WPcRel ## disp) { \
789    TestAdrpLdrPcRelLdr(kLdrWPcRelInsn, disp, adrp_offset, false, 0x12345678u, 0x1234u); \
790  }
791
792TEST_FOR_OFFSETS(LDRW_PCREL_LDR_TEST, 0x1234, 0x1238)
793
794// LDR <Xt>, <label> is aligned when offset + displacement is a multiple of 8.
795#define LDRX_PCREL_LDR_TEST(adrp_offset, disp) \
796  TEST_F(Arm64RelativePatcherTestDefault, StringBssEntry ## adrp_offset ## XPcRel ## disp) { \
797    bool unaligned = !IsAligned<8u>((adrp_offset) + 4u + static_cast<uint32_t>(disp)); \
798    bool has_thunk = ((adrp_offset) == 0xff8u || (adrp_offset) == 0xffcu) && unaligned; \
799    TestAdrpLdrPcRelLdr(kLdrXPcRelInsn, disp, adrp_offset, has_thunk, 0x12345678u, 0x1234u); \
800  }
801
802TEST_FOR_OFFSETS(LDRX_PCREL_LDR_TEST, 0x1234, 0x1238)
803
804// LDR <Wt>, [SP, #<pimm>] and LDR <Xt>, [SP, #<pimm>] are always aligned. No fixup needed.
805#define LDRW_SPREL_LDR_TEST(adrp_offset, disp) \
806  TEST_F(Arm64RelativePatcherTestDefault, StringBssEntry ## adrp_offset ## WSpRel ## disp) { \
807    TestAdrpLdrSpRelLdr(kLdrWSpRelInsn, (disp) >> 2, adrp_offset, false, 0x12345678u, 0x1234u); \
808  }
809
810TEST_FOR_OFFSETS(LDRW_SPREL_LDR_TEST, 0, 4)
811
812#define LDRX_SPREL_LDR_TEST(adrp_offset, disp) \
813  TEST_F(Arm64RelativePatcherTestDefault, StringBssEntry ## adrp_offset ## XSpRel ## disp) { \
814    TestAdrpLdrSpRelLdr(kLdrXSpRelInsn, (disp) >> 3, adrp_offset, false, 0x12345678u, 0x1234u); \
815  }
816
817TEST_FOR_OFFSETS(LDRX_SPREL_LDR_TEST, 0, 8)
818
819#define DEFAULT_LDUR_ADD_TEST(adrp_offset, disp) \
820  TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## Ldur ## disp) { \
821    bool has_thunk = ((adrp_offset) == 0xff8u || (adrp_offset) == 0xffcu); \
822    TestAdrpLdurAdd(adrp_offset, has_thunk, disp); \
823  }
824
825TEST_FOR_OFFSETS(DEFAULT_LDUR_ADD_TEST, 0x12345678, 0xffffc840)
826
827#define DENVER64_LDUR_ADD_TEST(adrp_offset, disp) \
828  TEST_F(Arm64RelativePatcherTestDenver64, StringReference ## adrp_offset ## Ldur ## disp) { \
829    TestAdrpLdurAdd(adrp_offset, false, disp); \
830  }
831
832TEST_FOR_OFFSETS(DENVER64_LDUR_ADD_TEST, 0x12345678, 0xffffc840)
833
834#define DEFAULT_SUBX3X2_ADD_TEST(adrp_offset, disp) \
835  TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## SubX3X2 ## disp) { \
836    /* SUB unrelated to "ADRP x0, addr". */ \
837    uint32_t sub = kSubXInsn | (100 << 10) | (2u << 5) | 3u;  /* SUB x3, x2, #100 */ \
838    TestAdrpInsn2Add(sub, adrp_offset, false, disp); \
839  }
840
841TEST_FOR_OFFSETS(DEFAULT_SUBX3X2_ADD_TEST, 0x12345678, 0xffffc840)
842
843#define DEFAULT_SUBSX3X0_ADD_TEST(adrp_offset, disp) \
844  TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## SubsX3X0 ## disp) { \
845    /* SUBS that uses the result of "ADRP x0, addr". */ \
846    uint32_t subs = kSubsXInsn | (100 << 10) | (0u << 5) | 3u;  /* SUBS x3, x0, #100 */ \
847    TestAdrpInsn2Add(subs, adrp_offset, false, disp); \
848  }
849
850TEST_FOR_OFFSETS(DEFAULT_SUBSX3X0_ADD_TEST, 0x12345678, 0xffffc840)
851
852#define DEFAULT_ADDX0X0_ADD_TEST(adrp_offset, disp) \
853  TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## AddX0X0 ## disp) { \
854    /* ADD that uses the result register of "ADRP x0, addr" as both source and destination. */ \
855    uint32_t add = kSubXInsn | (100 << 10) | (0u << 5) | 0u;  /* ADD x0, x0, #100 */ \
856    TestAdrpInsn2Add(add, adrp_offset, false, disp); \
857  }
858
859TEST_FOR_OFFSETS(DEFAULT_ADDX0X0_ADD_TEST, 0x12345678, 0xffffc840)
860
861#define DEFAULT_ADDSX0X2_ADD_TEST(adrp_offset, disp) \
862  TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## AddsX0X2 ## disp) { \
863    /* ADDS that does not use the result of "ADRP x0, addr" but overwrites that register. */ \
864    uint32_t adds = kAddsXInsn | (100 << 10) | (2u << 5) | 0u;  /* ADDS x0, x2, #100 */ \
865    bool has_thunk = ((adrp_offset) == 0xff8u || (adrp_offset) == 0xffcu); \
866    TestAdrpInsn2Add(adds, adrp_offset, has_thunk, disp); \
867  }
868
869TEST_FOR_OFFSETS(DEFAULT_ADDSX0X2_ADD_TEST, 0x12345678, 0xffffc840)
870
871// LDR <Wt>, <label> is always aligned. We should never have to use a fixup.
872#define LDRW_PCREL_ADD_TEST(adrp_offset, disp) \
873  TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## WPcRel ## disp) { \
874    TestAdrpLdrPcRelAdd(kLdrWPcRelInsn, disp, adrp_offset, false, 0x12345678u); \
875  }
876
877TEST_FOR_OFFSETS(LDRW_PCREL_ADD_TEST, 0x1234, 0x1238)
878
879// LDR <Xt>, <label> is aligned when offset + displacement is a multiple of 8.
880#define LDRX_PCREL_ADD_TEST(adrp_offset, disp) \
881  TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## XPcRel ## disp) { \
882    bool unaligned = !IsAligned<8u>((adrp_offset) + 4u + static_cast<uint32_t>(disp)); \
883    bool has_thunk = ((adrp_offset) == 0xff8u || (adrp_offset) == 0xffcu) && unaligned; \
884    TestAdrpLdrPcRelAdd(kLdrXPcRelInsn, disp, adrp_offset, has_thunk, 0x12345678u); \
885  }
886
887TEST_FOR_OFFSETS(LDRX_PCREL_ADD_TEST, 0x1234, 0x1238)
888
889// LDR <Wt>, [SP, #<pimm>] and LDR <Xt>, [SP, #<pimm>] are always aligned. No fixup needed.
890#define LDRW_SPREL_ADD_TEST(adrp_offset, disp) \
891  TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## WSpRel ## disp) { \
892    TestAdrpLdrSpRelAdd(kLdrWSpRelInsn, (disp) >> 2, adrp_offset, false, 0x12345678u); \
893  }
894
895TEST_FOR_OFFSETS(LDRW_SPREL_ADD_TEST, 0, 4)
896
897#define LDRX_SPREL_ADD_TEST(adrp_offset, disp) \
898  TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## XSpRel ## disp) { \
899    TestAdrpLdrSpRelAdd(kLdrXSpRelInsn, (disp) >> 3, adrp_offset, false, 0x12345678u); \
900  }
901
902TEST_FOR_OFFSETS(LDRX_SPREL_ADD_TEST, 0, 8)
903
904void Arm64RelativePatcherTest::TestBakerField(uint32_t offset, uint32_t ref_reg) {
905  uint32_t valid_regs[] = {
906      0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
907      10, 11, 12, 13, 14, 15,         18, 19,  // IP0 and IP1 are reserved.
908      20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
909      // LR and SP/ZR are reserved.
910  };
911  DCHECK_ALIGNED(offset, 4u);
912  DCHECK_LT(offset, 16 * KB);
913  constexpr size_t kMethodCodeSize = 8u;
914  constexpr size_t kLiteralOffset = 0u;
915  uint32_t method_idx = 0u;
916  for (uint32_t base_reg : valid_regs) {
917    for (uint32_t holder_reg : valid_regs) {
918      uint32_t ldr = kLdrWInsn | (offset << (10 - 2)) | (base_reg << 5) | ref_reg;
919      const std::vector<uint8_t> raw_code = RawCode({kCbnzIP1Plus0Insn, ldr});
920      ASSERT_EQ(kMethodCodeSize, raw_code.size());
921      ArrayRef<const uint8_t> code(raw_code);
922      uint32_t encoded_data =
923          Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg);
924      const LinkerPatch patches[] = {
925          LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
926      };
927      ++method_idx;
928      AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
929    }
930  }
931  Link();
932
933  // All thunks are at the end.
934  uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArm64Alignment);
935  method_idx = 0u;
936  for (uint32_t base_reg : valid_regs) {
937    for (uint32_t holder_reg : valid_regs) {
938      ++method_idx;
939      uint32_t cbnz_offset = thunk_offset - (GetMethodOffset(method_idx) + kLiteralOffset);
940      uint32_t cbnz = kCbnzIP1Plus0Insn | (cbnz_offset << (5 - 2));
941      uint32_t ldr = kLdrWInsn | (offset << (10 - 2)) | (base_reg << 5) | ref_reg;
942      const std::vector<uint8_t> expected_code = RawCode({cbnz, ldr});
943      ASSERT_EQ(kMethodCodeSize, expected_code.size());
944      ASSERT_TRUE(
945          CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
946
947      std::vector<uint8_t> expected_thunk = CompileBakerOffsetThunk(base_reg, holder_reg);
948      ASSERT_GT(output_.size(), thunk_offset);
949      ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
950      ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
951                                             expected_thunk.size());
952      if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
953        DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
954        ASSERT_TRUE(false);
955      }
956
957      size_t gray_check_offset = thunk_offset;
958      if (holder_reg == base_reg) {
959        // Verify that the null-check CBZ uses the correct register, i.e. holder_reg.
960        ASSERT_GE(output_.size() - gray_check_offset, 4u);
961        ASSERT_EQ(0x34000000u | holder_reg, GetOutputInsn(thunk_offset) & 0xff00001fu);
962        gray_check_offset +=4u;
963      }
964      // Verify that the lock word for gray bit check is loaded from the holder address.
965      static constexpr size_t kGrayCheckInsns = 5;
966      ASSERT_GE(output_.size() - gray_check_offset, 4u * kGrayCheckInsns);
967      const uint32_t load_lock_word =
968          kLdrWInsn |
969          (mirror::Object::MonitorOffset().Uint32Value() << (10 - 2)) |
970          (holder_reg << 5) |
971          /* ip0 */ 16;
972      EXPECT_EQ(load_lock_word, GetOutputInsn(gray_check_offset));
973      // Verify the gray bit check.
974      const uint32_t check_gray_bit_without_offset =
975          0x37000000u | (LockWord::kReadBarrierStateShift << 19) | /* ip0 */ 16;
976      EXPECT_EQ(check_gray_bit_without_offset, GetOutputInsn(gray_check_offset + 4u) & 0xfff8001fu);
977      // Verify the fake dependency.
978      const uint32_t fake_dependency =
979          0x8b408000u |             // ADD Xd, Xn, Xm, LSR 32
980          (/* ip0 */ 16 << 16) |    // Xm = ip0
981          (base_reg << 5) |         // Xn = base_reg
982          base_reg;                 // Xd = base_reg
983      EXPECT_EQ(fake_dependency, GetOutputInsn(gray_check_offset + 12u));
984      // Do not check the rest of the implementation.
985
986      // The next thunk follows on the next aligned offset.
987      thunk_offset += RoundUp(expected_thunk.size(), kArm64Alignment);
988    }
989  }
990}
991
992#define TEST_BAKER_FIELD(offset, ref_reg)     \
993  TEST_F(Arm64RelativePatcherTestDefault,     \
994    BakerOffset##offset##_##ref_reg) {        \
995    TestBakerField(offset, ref_reg);          \
996  }
997
998TEST_BAKER_FIELD(/* offset */ 0, /* ref_reg */ 0)
999TEST_BAKER_FIELD(/* offset */ 8, /* ref_reg */ 15)
1000TEST_BAKER_FIELD(/* offset */ 0x3ffc, /* ref_reg */ 29)
1001
1002TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkInTheMiddle) {
1003  // One thunk in the middle with maximum distance branches to it from both sides.
1004  // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
1005  constexpr uint32_t kLiteralOffset1 = 4;
1006  const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn});
1007  ArrayRef<const uint8_t> code1(raw_code1);
1008  uint32_t encoded_data =
1009      Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
1010  const LinkerPatch patches1[] = {
1011      LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
1012  };
1013  AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
1014
1015  // Allow thunk at 1MiB offset from the start of the method above. Literal offset being 4
1016  // allows the branch to reach that thunk.
1017  size_t filler1_size =
1018      1 * MB - RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArm64Alignment);
1019  std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 4u);
1020  ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
1021  AddCompiledMethod(MethodRef(2u), filler1_code);
1022
1023  // Enforce thunk reservation with a tiny method.
1024  AddCompiledMethod(MethodRef(3u), kNopCode);
1025
1026  // Allow reaching the thunk from the very beginning of a method 1MiB away. Backward branch
1027  // reaches the full 1MiB. Things to subtract:
1028  //   - thunk size and method 3 pre-header, rounded up (padding in between if needed)
1029  //   - method 3 code and method 4 pre-header, rounded up (padding in between if needed)
1030  //   - method 4 header (let there be no padding between method 4 code and method 5 pre-header).
1031  size_t thunk_size = CompileBakerOffsetThunk(/* base_reg */ 0, /* holder_reg */ 0).size();
1032  size_t filler2_size =
1033      1 * MB - RoundUp(thunk_size + sizeof(OatQuickMethodHeader), kArm64Alignment)
1034             - RoundUp(kNopCode.size() + sizeof(OatQuickMethodHeader), kArm64Alignment)
1035             - sizeof(OatQuickMethodHeader);
1036  std::vector<uint8_t> raw_filler2_code = GenNops(filler2_size / 4u);
1037  ArrayRef<const uint8_t> filler2_code(raw_filler2_code);
1038  AddCompiledMethod(MethodRef(4u), filler2_code);
1039
1040  constexpr uint32_t kLiteralOffset2 = 0;
1041  const std::vector<uint8_t> raw_code2 = RawCode({kCbnzIP1Plus0Insn, kLdrWInsn});
1042  ArrayRef<const uint8_t> code2(raw_code2);
1043  const LinkerPatch patches2[] = {
1044      LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset2, encoded_data),
1045  };
1046  AddCompiledMethod(MethodRef(5u), code2, ArrayRef<const LinkerPatch>(patches2));
1047
1048  Link();
1049
1050  uint32_t first_method_offset = GetMethodOffset(1u);
1051  uint32_t last_method_offset = GetMethodOffset(5u);
1052  EXPECT_EQ(2 * MB, last_method_offset - first_method_offset);
1053
1054  const uint32_t cbnz_max_forward = kCbnzIP1Plus0Insn | 0x007fffe0;
1055  const uint32_t cbnz_max_backward = kCbnzIP1Plus0Insn | 0x00800000;
1056  const std::vector<uint8_t> expected_code1 = RawCode({kNopInsn, cbnz_max_forward, kLdrWInsn});
1057  const std::vector<uint8_t> expected_code2 = RawCode({cbnz_max_backward, kLdrWInsn});
1058  ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
1059  ASSERT_TRUE(CheckLinkedMethod(MethodRef(5), ArrayRef<const uint8_t>(expected_code2)));
1060}
1061
1062TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkBeforeFiller) {
1063  // Based on the first part of BakerOffsetThunkInTheMiddle but the CBNZ is one instruction
1064  // earlier, so the thunk is emitted before the filler.
1065  // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
1066  constexpr uint32_t kLiteralOffset1 = 0;
1067  const std::vector<uint8_t> raw_code1 = RawCode({kCbnzIP1Plus0Insn, kLdrWInsn, kNopInsn});
1068  ArrayRef<const uint8_t> code1(raw_code1);
1069  uint32_t encoded_data =
1070      Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
1071  const LinkerPatch patches1[] = {
1072      LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
1073  };
1074  AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
1075
1076  // Allow thunk at 1MiB offset from the start of the method above. Literal offset being 4
1077  // allows the branch to reach that thunk.
1078  size_t filler1_size =
1079      1 * MB - RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArm64Alignment);
1080  std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 4u);
1081  ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
1082  AddCompiledMethod(MethodRef(2u), filler1_code);
1083
1084  Link();
1085
1086  const uint32_t cbnz_offset = RoundUp(raw_code1.size(), kArm64Alignment) - kLiteralOffset1;
1087  const uint32_t cbnz = kCbnzIP1Plus0Insn | (cbnz_offset << (5 - 2));
1088  const std::vector<uint8_t> expected_code1 = RawCode({cbnz, kLdrWInsn, kNopInsn});
1089  ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
1090}
1091
1092TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkInTheMiddleUnreachableFromLast) {
1093  // Based on the BakerOffsetThunkInTheMiddle but the CBNZ in the last method is preceded
1094  // by NOP and cannot reach the thunk in the middle, so we emit an extra thunk at the end.
1095  // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
1096  constexpr uint32_t kLiteralOffset1 = 4;
1097  const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn});
1098  ArrayRef<const uint8_t> code1(raw_code1);
1099  uint32_t encoded_data =
1100      Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
1101  const LinkerPatch patches1[] = {
1102      LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
1103  };
1104  AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
1105
1106  // Allow thunk at 1MiB offset from the start of the method above. Literal offset being 4
1107  // allows the branch to reach that thunk.
1108  size_t filler1_size =
1109      1 * MB - RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArm64Alignment);
1110  std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 4u);
1111  ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
1112  AddCompiledMethod(MethodRef(2u), filler1_code);
1113
1114  // Enforce thunk reservation with a tiny method.
1115  AddCompiledMethod(MethodRef(3u), kNopCode);
1116
1117  // If not for the extra NOP, this would allow reaching the thunk from the very beginning
1118  // of a method 1MiB away. Backward branch reaches the full 1MiB. Things to subtract:
1119  //   - thunk size and method 3 pre-header, rounded up (padding in between if needed)
1120  //   - method 3 code and method 4 pre-header, rounded up (padding in between if needed)
1121  //   - method 4 header (let there be no padding between method 4 code and method 5 pre-header).
1122  size_t thunk_size = CompileBakerOffsetThunk(/* base_reg */ 0, /* holder_reg */ 0).size();
1123  size_t filler2_size =
1124      1 * MB - RoundUp(thunk_size + sizeof(OatQuickMethodHeader), kArm64Alignment)
1125             - RoundUp(kNopCode.size() + sizeof(OatQuickMethodHeader), kArm64Alignment)
1126             - sizeof(OatQuickMethodHeader);
1127  std::vector<uint8_t> raw_filler2_code = GenNops(filler2_size / 4u);
1128  ArrayRef<const uint8_t> filler2_code(raw_filler2_code);
1129  AddCompiledMethod(MethodRef(4u), filler2_code);
1130
1131  // Extra NOP compared to BakerOffsetThunkInTheMiddle.
1132  constexpr uint32_t kLiteralOffset2 = 4;
1133  const std::vector<uint8_t> raw_code2 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn});
1134  ArrayRef<const uint8_t> code2(raw_code2);
1135  const LinkerPatch patches2[] = {
1136      LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset2, encoded_data),
1137  };
1138  AddCompiledMethod(MethodRef(5u), code2, ArrayRef<const LinkerPatch>(patches2));
1139
1140  Link();
1141
1142  const uint32_t cbnz_max_forward = kCbnzIP1Plus0Insn | 0x007fffe0;
1143  const uint32_t cbnz_last_offset = RoundUp(raw_code2.size(), kArm64Alignment) - kLiteralOffset2;
1144  const uint32_t cbnz_last = kCbnzIP1Plus0Insn | (cbnz_last_offset << (5 - 2));
1145  const std::vector<uint8_t> expected_code1 = RawCode({kNopInsn, cbnz_max_forward, kLdrWInsn});
1146  const std::vector<uint8_t> expected_code2 = RawCode({kNopInsn, cbnz_last, kLdrWInsn});
1147  ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
1148  ASSERT_TRUE(CheckLinkedMethod(MethodRef(5), ArrayRef<const uint8_t>(expected_code2)));
1149}
1150
1151TEST_F(Arm64RelativePatcherTestDefault, BakerArray) {
1152  uint32_t valid_regs[] = {
1153      0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
1154      10, 11, 12, 13, 14, 15,         18, 19,  // IP0 and IP1 are reserved.
1155      20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
1156      // LR and SP/ZR are reserved.
1157  };
1158  auto ldr = [](uint32_t base_reg) {
1159    uint32_t index_reg = (base_reg == 0u) ? 1u : 0u;
1160    uint32_t ref_reg = (base_reg == 2) ? 3u : 2u;
1161    return kLdrWLsl2Insn | (index_reg << 16) | (base_reg << 5) | ref_reg;
1162  };
1163  constexpr size_t kMethodCodeSize = 8u;
1164  constexpr size_t kLiteralOffset = 0u;
1165  uint32_t method_idx = 0u;
1166  for (uint32_t base_reg : valid_regs) {
1167    ++method_idx;
1168    const std::vector<uint8_t> raw_code = RawCode({kCbnzIP1Plus0Insn, ldr(base_reg)});
1169    ASSERT_EQ(kMethodCodeSize, raw_code.size());
1170    ArrayRef<const uint8_t> code(raw_code);
1171    const LinkerPatch patches[] = {
1172        LinkerPatch::BakerReadBarrierBranchPatch(
1173            kLiteralOffset, Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)),
1174    };
1175    AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1176  }
1177  Link();
1178
1179  // All thunks are at the end.
1180  uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArm64Alignment);
1181  method_idx = 0u;
1182  for (uint32_t base_reg : valid_regs) {
1183    ++method_idx;
1184    uint32_t cbnz_offset = thunk_offset - (GetMethodOffset(method_idx) + kLiteralOffset);
1185    uint32_t cbnz = kCbnzIP1Plus0Insn | (cbnz_offset << (5 - 2));
1186    const std::vector<uint8_t> expected_code = RawCode({cbnz, ldr(base_reg)});
1187    ASSERT_EQ(kMethodCodeSize, expected_code.size());
1188    EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1189
1190    std::vector<uint8_t> expected_thunk = CompileBakerArrayThunk(base_reg);
1191    ASSERT_GT(output_.size(), thunk_offset);
1192    ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1193    ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1194                                           expected_thunk.size());
1195    if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1196      DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1197      ASSERT_TRUE(false);
1198    }
1199
1200    // Verify that the lock word for gray bit check is loaded from the correct address
1201    // before the base_reg which points to the array data.
1202    static constexpr size_t kGrayCheckInsns = 5;
1203    ASSERT_GE(output_.size() - thunk_offset, 4u * kGrayCheckInsns);
1204    int32_t data_offset =
1205        mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value();
1206    int32_t offset = mirror::Object::MonitorOffset().Int32Value() - data_offset;
1207    ASSERT_LT(offset, 0);
1208    const uint32_t load_lock_word =
1209        kLdurWInsn |
1210        ((offset & 0x1ffu) << 12) |
1211        (base_reg << 5) |
1212        /* ip0 */ 16;
1213    EXPECT_EQ(load_lock_word, GetOutputInsn(thunk_offset));
1214    // Verify the gray bit check.
1215    const uint32_t check_gray_bit_without_offset =
1216        0x37000000u | (LockWord::kReadBarrierStateShift << 19) | /* ip0 */ 16;
1217    EXPECT_EQ(check_gray_bit_without_offset, GetOutputInsn(thunk_offset + 4u) & 0xfff8001fu);
1218    // Verify the fake dependency.
1219    const uint32_t fake_dependency =
1220        0x8b408000u |             // ADD Xd, Xn, Xm, LSR 32
1221        (/* ip0 */ 16 << 16) |    // Xm = ip0
1222        (base_reg << 5) |         // Xn = base_reg
1223        base_reg;                 // Xd = base_reg
1224    EXPECT_EQ(fake_dependency, GetOutputInsn(thunk_offset + 12u));
1225    // Do not check the rest of the implementation.
1226
1227    // The next thunk follows on the next aligned offset.
1228    thunk_offset += RoundUp(expected_thunk.size(), kArm64Alignment);
1229  }
1230}
1231
1232TEST_F(Arm64RelativePatcherTestDefault, BakerGcRoot) {
1233  uint32_t valid_regs[] = {
1234      0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
1235      10, 11, 12, 13, 14, 15,         18, 19,  // IP0 and IP1 are reserved.
1236      20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
1237      // LR and SP/ZR are reserved.
1238  };
1239  constexpr size_t kMethodCodeSize = 8u;
1240  constexpr size_t kLiteralOffset = 4u;
1241  uint32_t method_idx = 0u;
1242  for (uint32_t root_reg : valid_regs) {
1243    ++method_idx;
1244    uint32_t ldr = kLdrWInsn | (/* offset */ 8 << (10 - 2)) | (/* base_reg */ 0 << 5) | root_reg;
1245    const std::vector<uint8_t> raw_code = RawCode({ldr, kCbnzIP1Plus0Insn});
1246    ASSERT_EQ(kMethodCodeSize, raw_code.size());
1247    ArrayRef<const uint8_t> code(raw_code);
1248    const LinkerPatch patches[] = {
1249        LinkerPatch::BakerReadBarrierBranchPatch(
1250            kLiteralOffset, Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg)),
1251    };
1252    AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1253  }
1254  Link();
1255
1256  // All thunks are at the end.
1257  uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArm64Alignment);
1258  method_idx = 0u;
1259  for (uint32_t root_reg : valid_regs) {
1260    ++method_idx;
1261    uint32_t cbnz_offset = thunk_offset - (GetMethodOffset(method_idx) + kLiteralOffset);
1262    uint32_t cbnz = kCbnzIP1Plus0Insn | (cbnz_offset << (5 - 2));
1263    uint32_t ldr = kLdrWInsn | (/* offset */ 8 << (10 - 2)) | (/* base_reg */ 0 << 5) | root_reg;
1264    const std::vector<uint8_t> expected_code = RawCode({ldr, cbnz});
1265    ASSERT_EQ(kMethodCodeSize, expected_code.size());
1266    EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1267
1268    std::vector<uint8_t> expected_thunk = CompileBakerGcRootThunk(root_reg);
1269    ASSERT_GT(output_.size(), thunk_offset);
1270    ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1271    ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1272                                           expected_thunk.size());
1273    if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1274      DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1275      ASSERT_TRUE(false);
1276    }
1277
1278    // Verify that the fast-path null-check CBZ uses the correct register, i.e. root_reg.
1279    ASSERT_GE(output_.size() - thunk_offset, 4u);
1280    ASSERT_EQ(0x34000000u | root_reg, GetOutputInsn(thunk_offset) & 0xff00001fu);
1281    // Do not check the rest of the implementation.
1282
1283    // The next thunk follows on the next aligned offset.
1284    thunk_offset += RoundUp(expected_thunk.size(), kArm64Alignment);
1285  }
1286}
1287
1288TEST_F(Arm64RelativePatcherTestDefault, BakerAndMethodCallInteraction) {
1289  // During development, there was a `DCHECK_LE(MaxNextOffset(), next_thunk.MaxNextOffset());`
1290  // in `ArmBaseRelativePatcher::ThunkData::MakeSpaceBefore()` which does not necessarily
1291  // hold when we're reserving thunks of different sizes. This test exposes the situation
1292  // by using Baker thunks and a method call thunk.
1293
1294  // Add a method call patch that can reach to method 1 offset + 128MiB.
1295  uint32_t method_idx = 0u;
1296  constexpr size_t kMethodCallLiteralOffset = 4u;
1297  constexpr uint32_t kMissingMethodIdx = 2u;
1298  const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kBlPlus0});
1299  const LinkerPatch method1_patches[] = {
1300      LinkerPatch::RelativeCodePatch(kMethodCallLiteralOffset, nullptr, 2u),
1301  };
1302  ArrayRef<const uint8_t> code1(raw_code1);
1303  ++method_idx;
1304  AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(method1_patches));
1305
1306  // Skip kMissingMethodIdx.
1307  ++method_idx;
1308  ASSERT_EQ(kMissingMethodIdx, method_idx);
1309  // Add a method with the right size that the method code for the next one starts 1MiB
1310  // after code for method 1.
1311  size_t filler_size =
1312      1 * MB - RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArm64Alignment)
1313             - sizeof(OatQuickMethodHeader);
1314  std::vector<uint8_t> filler_code = GenNops(filler_size / 4u);
1315  ++method_idx;
1316  AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(filler_code));
1317  // Add 126 methods with 1MiB code+header, making the code for the next method start 1MiB
1318  // before the currently scheduled MaxNextOffset() for the method call thunk.
1319  for (uint32_t i = 0; i != 126; ++i) {
1320    filler_size = 1 * MB - sizeof(OatQuickMethodHeader);
1321    filler_code = GenNops(filler_size / 4u);
1322    ++method_idx;
1323    AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(filler_code));
1324  }
1325
1326  // Add 2 Baker GC root patches to the last method, one that would allow the thunk at
1327  // 1MiB + kArm64Alignment, i.e. kArm64Alignment after the method call thunk, and the
1328  // second that needs it kArm64Alignment after that. Given the size of the GC root thunk
1329  // is more than the space required by the method call thunk plus kArm64Alignment,
1330  // this pushes the first GC root thunk's pending MaxNextOffset() before the method call
1331  // thunk's pending MaxNextOffset() which needs to be adjusted.
1332  ASSERT_LT(RoundUp(CompileMethodCallThunk().size(), kArm64Alignment) + kArm64Alignment,
1333            CompileBakerGcRootThunk(/* root_reg */ 0).size());
1334  static_assert(kArm64Alignment == 16, "Code below assumes kArm64Alignment == 16");
1335  constexpr size_t kBakerLiteralOffset1 = 4u + kArm64Alignment;
1336  constexpr size_t kBakerLiteralOffset2 = 4u + 2 * kArm64Alignment;
1337  // Use offset = 0, base_reg = 0, the LDR is simply `kLdrWInsn | root_reg`.
1338  const uint32_t ldr1 = kLdrWInsn | /* root_reg */ 1;
1339  const uint32_t ldr2 = kLdrWInsn | /* root_reg */ 2;
1340  const std::vector<uint8_t> last_method_raw_code = RawCode({
1341      kNopInsn, kNopInsn, kNopInsn, kNopInsn,   // Padding before first GC root read barrier.
1342      ldr1, kCbnzIP1Plus0Insn,                  // First GC root LDR with read barrier.
1343      kNopInsn, kNopInsn,                       // Padding before second GC root read barrier.
1344      ldr2, kCbnzIP1Plus0Insn,                  // Second GC root LDR with read barrier.
1345  });
1346  uint32_t encoded_data1 = Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 1);
1347  uint32_t encoded_data2 = Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 2);
1348  const LinkerPatch last_method_patches[] = {
1349      LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1),
1350      LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2),
1351  };
1352  ++method_idx;
1353  AddCompiledMethod(MethodRef(method_idx),
1354                    ArrayRef<const uint8_t>(last_method_raw_code),
1355                    ArrayRef<const LinkerPatch>(last_method_patches));
1356
1357  // The main purpose of the test is to check that Link() does not cause a crash.
1358  Link();
1359
1360  ASSERT_EQ(127 * MB, GetMethodOffset(method_idx) - GetMethodOffset(1u));
1361}
1362
1363}  // namespace linker
1364}  // namespace art
1365