quick_entrypoints_arm64.S revision 3b7537bfc5a6b7ccb18b3970d8edf14b72464af7
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "asm_support_arm64.S" 18 19#include "arch/quick_alloc_entrypoints.S" 20 21 22.macro INCREASE_FRAME frame_adjustment 23 sub sp, sp, #(\frame_adjustment) 24 .cfi_adjust_cfa_offset (\frame_adjustment) 25.endm 26 27.macro DECREASE_FRAME frame_adjustment 28 add sp, sp, #(\frame_adjustment) 29 .cfi_adjust_cfa_offset -(\frame_adjustment) 30.endm 31 32.macro SAVE_REG reg, offset 33 str \reg, [sp, #(\offset)] 34 .cfi_rel_offset \reg, (\offset) 35.endm 36 37.macro RESTORE_REG reg, offset 38 ldr \reg, [sp, #(\offset)] 39 .cfi_restore \reg 40.endm 41 42.macro SAVE_TWO_REGS reg1, reg2, offset 43 stp \reg1, \reg2, [sp, #(\offset)] 44 .cfi_rel_offset \reg1, (\offset) 45 .cfi_rel_offset \reg2, (\offset) + 8 46.endm 47 48.macro RESTORE_TWO_REGS reg1, reg2, offset 49 ldp \reg1, \reg2, [sp, #(\offset)] 50 .cfi_restore \reg1 51 .cfi_restore \reg2 52.endm 53 54.macro SAVE_TWO_REGS_INCREASE_FRAME reg1, reg2, frame_adjustment 55 stp \reg1, \reg2, [sp, #-(\frame_adjustment)]! 56 .cfi_adjust_cfa_offset (\frame_adjustment) 57 .cfi_rel_offset \reg1, 0 58 .cfi_rel_offset \reg2, 8 59.endm 60 61.macro RESTORE_TWO_REGS_DECREASE_FRAME reg1, reg2, frame_adjustment 62 ldp \reg1, \reg2, [sp], #(\frame_adjustment) 63 .cfi_restore \reg1 64 .cfi_restore \reg2 65 .cfi_adjust_cfa_offset -(\frame_adjustment) 66.endm 67 68 /* 69 * Macro that sets up the callee save frame to conform with 70 * Runtime::CreateCalleeSaveMethod(kSaveAllCalleeSaves) 71 */ 72.macro SETUP_SAVE_ALL_CALLEE_SAVES_FRAME 73 // art::Runtime** xIP0 = &art::Runtime::instance_ 74 adrp xIP0, :got:_ZN3art7Runtime9instance_E 75 ldr xIP0, [xIP0, #:got_lo12:_ZN3art7Runtime9instance_E] 76 77 // Our registers aren't intermixed - just spill in order. 78 ldr xIP0, [xIP0] // art::Runtime* xIP0 = art::Runtime::instance_; 79 80 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveAllCalleeSaves]; 81 ldr xIP0, [xIP0, RUNTIME_SAVE_ALL_CALLEE_SAVES_METHOD_OFFSET] 82 83 INCREASE_FRAME 176 84 85 // Ugly compile-time check, but we only have the preprocessor. 86#if (FRAME_SIZE_SAVE_ALL_CALLEE_SAVES != 176) 87#error "FRAME_SIZE_SAVE_ALL_CALLEE_SAVES(ARM64) size not as expected." 88#endif 89 90 // Stack alignment filler [sp, #8]. 91 // FP callee-saves. 92 stp d8, d9, [sp, #16] 93 stp d10, d11, [sp, #32] 94 stp d12, d13, [sp, #48] 95 stp d14, d15, [sp, #64] 96 97 // GP callee-saves 98 SAVE_TWO_REGS x19, x20, 80 99 SAVE_TWO_REGS x21, x22, 96 100 SAVE_TWO_REGS x23, x24, 112 101 SAVE_TWO_REGS x25, x26, 128 102 SAVE_TWO_REGS x27, x28, 144 103 SAVE_TWO_REGS x29, xLR, 160 104 105 // Store ArtMethod* Runtime::callee_save_methods_[kSaveAllCalleeSaves]. 106 str xIP0, [sp] 107 // Place sp in Thread::Current()->top_quick_frame. 108 mov xIP0, sp 109 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 110.endm 111 112 /* 113 * Macro that sets up the callee save frame to conform with 114 * Runtime::CreateCalleeSaveMethod(kSaveRefsOnly). 115 */ 116.macro SETUP_SAVE_REFS_ONLY_FRAME 117 // art::Runtime** xIP0 = &art::Runtime::instance_ 118 adrp xIP0, :got:_ZN3art7Runtime9instance_E 119 ldr xIP0, [xIP0, #:got_lo12:_ZN3art7Runtime9instance_E] 120 121 // Our registers aren't intermixed - just spill in order. 122 ldr xIP0, [xIP0] // art::Runtime* xIP0 = art::Runtime::instance_; 123 124 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveRefOnly]; 125 ldr xIP0, [xIP0, RUNTIME_SAVE_REFS_ONLY_METHOD_OFFSET] 126 127 INCREASE_FRAME 96 128 129 // Ugly compile-time check, but we only have the preprocessor. 130#if (FRAME_SIZE_SAVE_REFS_ONLY != 96) 131#error "FRAME_SIZE_SAVE_REFS_ONLY(ARM64) size not as expected." 132#endif 133 134 // GP callee-saves. 135 // x20 paired with ArtMethod* - see below. 136 SAVE_TWO_REGS x21, x22, 16 137 SAVE_TWO_REGS x23, x24, 32 138 SAVE_TWO_REGS x25, x26, 48 139 SAVE_TWO_REGS x27, x28, 64 140 SAVE_TWO_REGS x29, xLR, 80 141 142 // Store ArtMethod* Runtime::callee_save_methods_[kSaveRefsOnly]. 143 stp xIP0, x20, [sp] 144 .cfi_rel_offset x20, 8 145 146 // Place sp in Thread::Current()->top_quick_frame. 147 mov xIP0, sp 148 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 149.endm 150 151// TODO: Probably no need to restore registers preserved by aapcs64. 152.macro RESTORE_SAVE_REFS_ONLY_FRAME 153 // Callee-saves. 154 RESTORE_REG x20, 8 155 RESTORE_TWO_REGS x21, x22, 16 156 RESTORE_TWO_REGS x23, x24, 32 157 RESTORE_TWO_REGS x25, x26, 48 158 RESTORE_TWO_REGS x27, x28, 64 159 RESTORE_TWO_REGS x29, xLR, 80 160 161 DECREASE_FRAME 96 162.endm 163 164.macro POP_SAVE_REFS_ONLY_FRAME 165 DECREASE_FRAME 96 166.endm 167 168.macro RESTORE_SAVE_REFS_ONLY_FRAME_AND_RETURN 169 RESTORE_SAVE_REFS_ONLY_FRAME 170 ret 171.endm 172 173 174.macro SETUP_SAVE_REFS_AND_ARGS_FRAME_INTERNAL 175 INCREASE_FRAME 224 176 177 // Ugly compile-time check, but we only have the preprocessor. 178#if (FRAME_SIZE_SAVE_REFS_AND_ARGS != 224) 179#error "FRAME_SIZE_SAVE_REFS_AND_ARGS(ARM64) size not as expected." 180#endif 181 182 // Stack alignment filler [sp, #8]. 183 // FP args. 184 stp d0, d1, [sp, #16] 185 stp d2, d3, [sp, #32] 186 stp d4, d5, [sp, #48] 187 stp d6, d7, [sp, #64] 188 189 // Core args. 190 SAVE_TWO_REGS x1, x2, 80 191 SAVE_TWO_REGS x3, x4, 96 192 SAVE_TWO_REGS x5, x6, 112 193 194 // x7, Callee-saves. 195 SAVE_TWO_REGS x7, x20, 128 196 SAVE_TWO_REGS x21, x22, 144 197 SAVE_TWO_REGS x23, x24, 160 198 SAVE_TWO_REGS x25, x26, 176 199 SAVE_TWO_REGS x27, x28, 192 200 201 // x29(callee-save) and LR. 202 SAVE_TWO_REGS x29, xLR, 208 203 204.endm 205 206 /* 207 * Macro that sets up the callee save frame to conform with 208 * Runtime::CreateCalleeSaveMethod(kSaveRefsAndArgs). 209 * 210 * TODO This is probably too conservative - saving FP & LR. 211 */ 212.macro SETUP_SAVE_REFS_AND_ARGS_FRAME 213 // art::Runtime** xIP0 = &art::Runtime::instance_ 214 adrp xIP0, :got:_ZN3art7Runtime9instance_E 215 ldr xIP0, [xIP0, #:got_lo12:_ZN3art7Runtime9instance_E] 216 217 // Our registers aren't intermixed - just spill in order. 218 ldr xIP0, [xIP0] // art::Runtime* xIP0 = art::Runtime::instance_; 219 220 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveRefAndArgs]; 221 ldr xIP0, [xIP0, RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET] 222 223 SETUP_SAVE_REFS_AND_ARGS_FRAME_INTERNAL 224 225 str xIP0, [sp] // Store ArtMethod* Runtime::callee_save_methods_[kSaveRefsAndArgs]. 226 // Place sp in Thread::Current()->top_quick_frame. 227 mov xIP0, sp 228 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 229.endm 230 231.macro SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_X0 232 SETUP_SAVE_REFS_AND_ARGS_FRAME_INTERNAL 233 str x0, [sp, #0] // Store ArtMethod* to bottom of stack. 234 // Place sp in Thread::Current()->top_quick_frame. 235 mov xIP0, sp 236 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 237.endm 238 239// TODO: Probably no need to restore registers preserved by aapcs64. 240.macro RESTORE_SAVE_REFS_AND_ARGS_FRAME 241 // FP args. 242 ldp d0, d1, [sp, #16] 243 ldp d2, d3, [sp, #32] 244 ldp d4, d5, [sp, #48] 245 ldp d6, d7, [sp, #64] 246 247 // Core args. 248 RESTORE_TWO_REGS x1, x2, 80 249 RESTORE_TWO_REGS x3, x4, 96 250 RESTORE_TWO_REGS x5, x6, 112 251 252 // x7, Callee-saves. 253 RESTORE_TWO_REGS x7, x20, 128 254 RESTORE_TWO_REGS x21, x22, 144 255 RESTORE_TWO_REGS x23, x24, 160 256 RESTORE_TWO_REGS x25, x26, 176 257 RESTORE_TWO_REGS x27, x28, 192 258 259 // x29(callee-save) and LR. 260 RESTORE_TWO_REGS x29, xLR, 208 261 262 DECREASE_FRAME 224 263.endm 264 265 /* 266 * Macro that sets up the callee save frame to conform with 267 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 268 * when the SP has already been decremented by FRAME_SIZE_SAVE_EVERYTHING 269 * and saving registers x29 and LR is handled elsewhere. 270 */ 271.macro SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR 272 // Ugly compile-time check, but we only have the preprocessor. 273#if (FRAME_SIZE_SAVE_EVERYTHING != 512) 274#error "FRAME_SIZE_SAVE_EVERYTHING(ARM64) size not as expected." 275#endif 276 277 // Save FP registers. 278 // For better performance, store d0 and d31 separately, so that all STPs are 16-byte aligned. 279 str d0, [sp, #8] 280 stp d1, d2, [sp, #16] 281 stp d3, d4, [sp, #32] 282 stp d5, d6, [sp, #48] 283 stp d7, d8, [sp, #64] 284 stp d9, d10, [sp, #80] 285 stp d11, d12, [sp, #96] 286 stp d13, d14, [sp, #112] 287 stp d15, d16, [sp, #128] 288 stp d17, d18, [sp, #144] 289 stp d19, d20, [sp, #160] 290 stp d21, d22, [sp, #176] 291 stp d23, d24, [sp, #192] 292 stp d25, d26, [sp, #208] 293 stp d27, d28, [sp, #224] 294 stp d29, d30, [sp, #240] 295 str d31, [sp, #256] 296 297 // Save core registers. 298 SAVE_REG x0, 264 299 SAVE_TWO_REGS x1, x2, 272 300 SAVE_TWO_REGS x3, x4, 288 301 SAVE_TWO_REGS x5, x6, 304 302 SAVE_TWO_REGS x7, x8, 320 303 SAVE_TWO_REGS x9, x10, 336 304 SAVE_TWO_REGS x11, x12, 352 305 SAVE_TWO_REGS x13, x14, 368 306 SAVE_TWO_REGS x15, x16, 384 307 SAVE_TWO_REGS x17, x18, 400 308 SAVE_TWO_REGS x19, x20, 416 309 SAVE_TWO_REGS x21, x22, 432 310 SAVE_TWO_REGS x23, x24, 448 311 SAVE_TWO_REGS x25, x26, 464 312 SAVE_TWO_REGS x27, x28, 480 313 314 // art::Runtime** xIP0 = &art::Runtime::instance_ 315 adrp xIP0, :got:_ZN3art7Runtime9instance_E 316 ldr xIP0, [xIP0, #:got_lo12:_ZN3art7Runtime9instance_E] 317 318 ldr xIP0, [xIP0] // art::Runtime* xIP0 = art::Runtime::instance_; 319 320 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveEverything]; 321 ldr xIP0, [xIP0, RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET] 322 323 // Store ArtMethod* Runtime::callee_save_methods_[kSaveEverything]. 324 str xIP0, [sp] 325 // Place sp in Thread::Current()->top_quick_frame. 326 mov xIP0, sp 327 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 328.endm 329 330 /* 331 * Macro that sets up the callee save frame to conform with 332 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 333 */ 334.macro SETUP_SAVE_EVERYTHING_FRAME 335 INCREASE_FRAME 512 336 SAVE_TWO_REGS x29, xLR, 496 337 SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR 338.endm 339 340.macro RESTORE_SAVE_EVERYTHING_FRAME 341 // Restore FP registers. 342 // For better performance, load d0 and d31 separately, so that all LDPs are 16-byte aligned. 343 ldr d0, [sp, #8] 344 ldp d1, d2, [sp, #16] 345 ldp d3, d4, [sp, #32] 346 ldp d5, d6, [sp, #48] 347 ldp d7, d8, [sp, #64] 348 ldp d9, d10, [sp, #80] 349 ldp d11, d12, [sp, #96] 350 ldp d13, d14, [sp, #112] 351 ldp d15, d16, [sp, #128] 352 ldp d17, d18, [sp, #144] 353 ldp d19, d20, [sp, #160] 354 ldp d21, d22, [sp, #176] 355 ldp d23, d24, [sp, #192] 356 ldp d25, d26, [sp, #208] 357 ldp d27, d28, [sp, #224] 358 ldp d29, d30, [sp, #240] 359 ldr d31, [sp, #256] 360 361 // Restore core registers. 362 RESTORE_REG x0, 264 363 RESTORE_TWO_REGS x1, x2, 272 364 RESTORE_TWO_REGS x3, x4, 288 365 RESTORE_TWO_REGS x5, x6, 304 366 RESTORE_TWO_REGS x7, x8, 320 367 RESTORE_TWO_REGS x9, x10, 336 368 RESTORE_TWO_REGS x11, x12, 352 369 RESTORE_TWO_REGS x13, x14, 368 370 RESTORE_TWO_REGS x15, x16, 384 371 RESTORE_TWO_REGS x17, x18, 400 372 RESTORE_TWO_REGS x19, x20, 416 373 RESTORE_TWO_REGS x21, x22, 432 374 RESTORE_TWO_REGS x23, x24, 448 375 RESTORE_TWO_REGS x25, x26, 464 376 RESTORE_TWO_REGS x27, x28, 480 377 RESTORE_TWO_REGS x29, xLR, 496 378 379 DECREASE_FRAME 512 380.endm 381 382.macro RETURN_IF_RESULT_IS_ZERO 383 cbnz x0, 1f // result non-zero branch over 384 ret // return 3851: 386.endm 387 388.macro RETURN_IF_RESULT_IS_NON_ZERO 389 cbz x0, 1f // result zero branch over 390 ret // return 3911: 392.endm 393 394 /* 395 * Macro that set calls through to artDeliverPendingExceptionFromCode, where the pending 396 * exception is Thread::Current()->exception_ 397 */ 398.macro DELIVER_PENDING_EXCEPTION 399 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME 400 mov x0, xSELF 401 402 // Point of no return. 403 b artDeliverPendingExceptionFromCode // artDeliverPendingExceptionFromCode(Thread*) 404 brk 0 // Unreached 405.endm 406 407.macro RETURN_OR_DELIVER_PENDING_EXCEPTION_REG reg 408 ldr \reg, [xSELF, # THREAD_EXCEPTION_OFFSET] // Get exception field. 409 cbnz \reg, 1f 410 ret 4111: 412 DELIVER_PENDING_EXCEPTION 413.endm 414 415.macro RETURN_OR_DELIVER_PENDING_EXCEPTION 416 RETURN_OR_DELIVER_PENDING_EXCEPTION_REG xIP0 417.endm 418 419// Same as above with x1. This is helpful in stubs that want to avoid clobbering another register. 420.macro RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 421 RETURN_OR_DELIVER_PENDING_EXCEPTION_REG x1 422.endm 423 424.macro RETURN_IF_W0_IS_ZERO_OR_DELIVER 425 cbnz w0, 1f // result non-zero branch over 426 ret // return 4271: 428 DELIVER_PENDING_EXCEPTION 429.endm 430 431.macro NO_ARG_RUNTIME_EXCEPTION c_name, cxx_name 432 .extern \cxx_name 433ENTRY \c_name 434 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context 435 mov x0, xSELF // pass Thread::Current 436 b \cxx_name // \cxx_name(Thread*) 437END \c_name 438.endm 439 440.macro ONE_ARG_RUNTIME_EXCEPTION c_name, cxx_name 441 .extern \cxx_name 442ENTRY \c_name 443 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context. 444 mov x1, xSELF // pass Thread::Current. 445 b \cxx_name // \cxx_name(arg, Thread*). 446 brk 0 447END \c_name 448.endm 449 450.macro TWO_ARG_RUNTIME_EXCEPTION c_name, cxx_name 451 .extern \cxx_name 452ENTRY \c_name 453 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context 454 mov x2, xSELF // pass Thread::Current 455 b \cxx_name // \cxx_name(arg1, arg2, Thread*) 456 brk 0 457END \c_name 458.endm 459 460 /* 461 * Called by managed code, saves callee saves and then calls artThrowException 462 * that will place a mock Method* at the bottom of the stack. Arg1 holds the exception. 463 */ 464ONE_ARG_RUNTIME_EXCEPTION art_quick_deliver_exception, artDeliverExceptionFromCode 465 466 /* 467 * Called by managed code to create and deliver a NullPointerException. 468 */ 469NO_ARG_RUNTIME_EXCEPTION art_quick_throw_null_pointer_exception, artThrowNullPointerExceptionFromCode 470 471 /* 472 * Call installed by a signal handler to create and deliver a NullPointerException. 473 */ 474 .extern art_quick_throw_null_pointer_exception_from_signal 475ENTRY art_quick_throw_null_pointer_exception_from_signal 476 // The fault handler pushes the gc map address, i.e. "return address", to stack 477 // and passes the fault address in LR. So we need to set up the CFI info accordingly. 478 .cfi_def_cfa_offset __SIZEOF_POINTER__ 479 .cfi_rel_offset lr, 0 480 // Save all registers as basis for long jump context. 481 INCREASE_FRAME (FRAME_SIZE_SAVE_EVERYTHING - __SIZEOF_POINTER__) 482 SAVE_REG x29, (FRAME_SIZE_SAVE_EVERYTHING - 2 * __SIZEOF_POINTER__) // LR already saved. 483 SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR 484 mov x0, lr // pass the fault address stored in LR by the fault handler. 485 mov x1, xSELF // pass Thread::Current. 486 // TODO: Change other throwing entrypoints to use BL instead of B. http://b/31468464 487 bl artThrowNullPointerExceptionFromSignal // (arg, Thread*). 488 brk 0 489END art_quick_throw_null_pointer_exception_from_signal 490 491 /* 492 * Called by managed code to create and deliver an ArithmeticException. 493 */ 494NO_ARG_RUNTIME_EXCEPTION art_quick_throw_div_zero, artThrowDivZeroFromCode 495 496 /* 497 * Called by managed code to create and deliver an ArrayIndexOutOfBoundsException. Arg1 holds 498 * index, arg2 holds limit. 499 */ 500TWO_ARG_RUNTIME_EXCEPTION art_quick_throw_array_bounds, artThrowArrayBoundsFromCode 501 502 /* 503 * Called by managed code to create and deliver a StringIndexOutOfBoundsException 504 * as if thrown from a call to String.charAt(). Arg1 holds index, arg2 holds limit. 505 */ 506TWO_ARG_RUNTIME_EXCEPTION art_quick_throw_string_bounds, artThrowStringBoundsFromCode 507 508 /* 509 * Called by managed code to create and deliver a StackOverflowError. 510 */ 511NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFromCode 512 513 /* 514 * Called by managed code to create and deliver a NoSuchMethodError. 515 */ 516ONE_ARG_RUNTIME_EXCEPTION art_quick_throw_no_such_method, artThrowNoSuchMethodFromCode 517 518 /* 519 * All generated callsites for interface invokes and invocation slow paths will load arguments 520 * as usual - except instead of loading arg0/x0 with the target Method*, arg0/x0 will contain 521 * the method_idx. This wrapper will save arg1-arg3, and call the appropriate C helper. 522 * NOTE: "this" is first visible argument of the target, and so can be found in arg1/x1. 523 * 524 * The helper will attempt to locate the target and return a 128-bit result in x0/x1 consisting 525 * of the target Method* in x0 and method->code_ in x1. 526 * 527 * If unsuccessful, the helper will return null/????. There will be a pending exception in the 528 * thread and we branch to another stub to deliver it. 529 * 530 * On success this wrapper will restore arguments and *jump* to the target, leaving the lr 531 * pointing back to the original caller. 532 * 533 * Adapted from ARM32 code. 534 * 535 * Clobbers xIP0. 536 */ 537.macro INVOKE_TRAMPOLINE_BODY cxx_name 538 .extern \cxx_name 539 SETUP_SAVE_REFS_AND_ARGS_FRAME // save callee saves in case allocation triggers GC 540 // Helper signature is always 541 // (method_idx, *this_object, *caller_method, *self, sp) 542 543 mov x2, xSELF // pass Thread::Current 544 mov x3, sp 545 bl \cxx_name // (method_idx, this, Thread*, SP) 546 mov xIP0, x1 // save Method*->code_ 547 RESTORE_SAVE_REFS_AND_ARGS_FRAME 548 cbz x0, 1f // did we find the target? if not go to exception delivery 549 br xIP0 // tail call to target 5501: 551 DELIVER_PENDING_EXCEPTION 552.endm 553.macro INVOKE_TRAMPOLINE c_name, cxx_name 554ENTRY \c_name 555 INVOKE_TRAMPOLINE_BODY \cxx_name 556END \c_name 557.endm 558 559INVOKE_TRAMPOLINE art_quick_invoke_interface_trampoline_with_access_check, artInvokeInterfaceTrampolineWithAccessCheck 560 561INVOKE_TRAMPOLINE art_quick_invoke_static_trampoline_with_access_check, artInvokeStaticTrampolineWithAccessCheck 562INVOKE_TRAMPOLINE art_quick_invoke_direct_trampoline_with_access_check, artInvokeDirectTrampolineWithAccessCheck 563INVOKE_TRAMPOLINE art_quick_invoke_super_trampoline_with_access_check, artInvokeSuperTrampolineWithAccessCheck 564INVOKE_TRAMPOLINE art_quick_invoke_virtual_trampoline_with_access_check, artInvokeVirtualTrampolineWithAccessCheck 565 566 567.macro INVOKE_STUB_CREATE_FRAME 568 569SAVE_SIZE=15*8 // x4, x5, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, SP, LR, FP saved. 570SAVE_SIZE_AND_METHOD=SAVE_SIZE+8 571 572 573 mov x9, sp // Save stack pointer. 574 .cfi_register sp,x9 575 576 add x10, x2, # SAVE_SIZE_AND_METHOD // calculate size of frame. 577 sub x10, sp, x10 // Calculate SP position - saves + ArtMethod* + args 578 and x10, x10, # ~0xf // Enforce 16 byte stack alignment. 579 mov sp, x10 // Set new SP. 580 581 sub x10, x9, #SAVE_SIZE // Calculate new FP (later). Done here as we must move SP 582 .cfi_def_cfa_register x10 // before this. 583 .cfi_adjust_cfa_offset SAVE_SIZE 584 585 str x28, [x10, #112] 586 .cfi_rel_offset x28, 112 587 588 stp x26, x27, [x10, #96] 589 .cfi_rel_offset x26, 96 590 .cfi_rel_offset x27, 104 591 592 stp x24, x25, [x10, #80] 593 .cfi_rel_offset x24, 80 594 .cfi_rel_offset x25, 88 595 596 stp x22, x23, [x10, #64] 597 .cfi_rel_offset x22, 64 598 .cfi_rel_offset x23, 72 599 600 stp x20, x21, [x10, #48] 601 .cfi_rel_offset x20, 48 602 .cfi_rel_offset x21, 56 603 604 stp x9, x19, [x10, #32] // Save old stack pointer and x19. 605 .cfi_rel_offset sp, 32 606 .cfi_rel_offset x19, 40 607 608 stp x4, x5, [x10, #16] // Save result and shorty addresses. 609 .cfi_rel_offset x4, 16 610 .cfi_rel_offset x5, 24 611 612 stp xFP, xLR, [x10] // Store LR & FP. 613 .cfi_rel_offset x29, 0 614 .cfi_rel_offset x30, 8 615 616 mov xFP, x10 // Use xFP now, as it's callee-saved. 617 .cfi_def_cfa_register x29 618 mov xSELF, x3 // Move thread pointer into SELF register. 619 620 // Copy arguments into stack frame. 621 // Use simple copy routine for now. 622 // 4 bytes per slot. 623 // X1 - source address 624 // W2 - args length 625 // X9 - destination address. 626 // W10 - temporary 627 add x9, sp, #8 // Destination address is bottom of stack + null. 628 629 // Copy parameters into the stack. Use numeric label as this is a macro and Clang's assembler 630 // does not have unique-id variables. 6311: 632 cmp w2, #0 633 beq 2f 634 sub w2, w2, #4 // Need 65536 bytes of range. 635 ldr w10, [x1, x2] 636 str w10, [x9, x2] 637 638 b 1b 639 6402: 641 // Store null into ArtMethod* at bottom of frame. 642 str xzr, [sp] 643.endm 644 645.macro INVOKE_STUB_CALL_AND_RETURN 646 647 // load method-> METHOD_QUICK_CODE_OFFSET 648 ldr x9, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 649 // Branch to method. 650 blr x9 651 652 // Restore return value address and shorty address. 653 ldp x4,x5, [xFP, #16] 654 .cfi_restore x4 655 .cfi_restore x5 656 657 ldr x28, [xFP, #112] 658 .cfi_restore x28 659 660 ldp x26, x27, [xFP, #96] 661 .cfi_restore x26 662 .cfi_restore x27 663 664 ldp x24, x25, [xFP, #80] 665 .cfi_restore x24 666 .cfi_restore x25 667 668 ldp x22, x23, [xFP, #64] 669 .cfi_restore x22 670 .cfi_restore x23 671 672 ldp x20, x21, [xFP, #48] 673 .cfi_restore x20 674 .cfi_restore x21 675 676 // Store result (w0/x0/s0/d0) appropriately, depending on resultType. 677 ldrb w10, [x5] 678 679 // Check the return type and store the correct register into the jvalue in memory. 680 // Use numeric label as this is a macro and Clang's assembler does not have unique-id variables. 681 682 // Don't set anything for a void type. 683 cmp w10, #'V' 684 beq 3f 685 686 // Is it a double? 687 cmp w10, #'D' 688 bne 1f 689 str d0, [x4] 690 b 3f 691 6921: // Is it a float? 693 cmp w10, #'F' 694 bne 2f 695 str s0, [x4] 696 b 3f 697 6982: // Just store x0. Doesn't matter if it is 64 or 32 bits. 699 str x0, [x4] 700 7013: // Finish up. 702 ldp x2, x19, [xFP, #32] // Restore stack pointer and x19. 703 .cfi_restore x19 704 mov sp, x2 705 .cfi_restore sp 706 707 ldp xFP, xLR, [xFP] // Restore old frame pointer and link register. 708 .cfi_restore x29 709 .cfi_restore x30 710 711 ret 712 713.endm 714 715 716/* 717 * extern"C" void art_quick_invoke_stub(ArtMethod *method, x0 718 * uint32_t *args, x1 719 * uint32_t argsize, w2 720 * Thread *self, x3 721 * JValue *result, x4 722 * char *shorty); x5 723 * +----------------------+ 724 * | | 725 * | C/C++ frame | 726 * | LR'' | 727 * | FP'' | <- SP' 728 * +----------------------+ 729 * +----------------------+ 730 * | x28 | <- TODO: Remove callee-saves. 731 * | : | 732 * | x19 | 733 * | SP' | 734 * | X5 | 735 * | X4 | Saved registers 736 * | LR' | 737 * | FP' | <- FP 738 * +----------------------+ 739 * | uint32_t out[n-1] | 740 * | : : | Outs 741 * | uint32_t out[0] | 742 * | ArtMethod* | <- SP value=null 743 * +----------------------+ 744 * 745 * Outgoing registers: 746 * x0 - Method* 747 * x1-x7 - integer parameters. 748 * d0-d7 - Floating point parameters. 749 * xSELF = self 750 * SP = & of ArtMethod* 751 * x1 = "this" pointer. 752 * 753 */ 754ENTRY art_quick_invoke_stub 755 // Spill registers as per AACPS64 calling convention. 756 INVOKE_STUB_CREATE_FRAME 757 758 // Fill registers x/w1 to x/w7 and s/d0 to s/d7 with parameters. 759 // Parse the passed shorty to determine which register to load. 760 // Load addresses for routines that load WXSD registers. 761 adr x11, .LstoreW2 762 adr x12, .LstoreX2 763 adr x13, .LstoreS0 764 adr x14, .LstoreD0 765 766 // Initialize routine offsets to 0 for integers and floats. 767 // x8 for integers, x15 for floating point. 768 mov x8, #0 769 mov x15, #0 770 771 add x10, x5, #1 // Load shorty address, plus one to skip return value. 772 ldr w1, [x9],#4 // Load "this" parameter, and increment arg pointer. 773 774 // Loop to fill registers. 775.LfillRegisters: 776 ldrb w17, [x10], #1 // Load next character in signature, and increment. 777 cbz w17, .LcallFunction // Exit at end of signature. Shorty 0 terminated. 778 779 cmp w17, #'F' // is this a float? 780 bne .LisDouble 781 782 cmp x15, # 8*12 // Skip this load if all registers full. 783 beq .Ladvance4 784 785 add x17, x13, x15 // Calculate subroutine to jump to. 786 br x17 787 788.LisDouble: 789 cmp w17, #'D' // is this a double? 790 bne .LisLong 791 792 cmp x15, # 8*12 // Skip this load if all registers full. 793 beq .Ladvance8 794 795 add x17, x14, x15 // Calculate subroutine to jump to. 796 br x17 797 798.LisLong: 799 cmp w17, #'J' // is this a long? 800 bne .LisOther 801 802 cmp x8, # 6*12 // Skip this load if all registers full. 803 beq .Ladvance8 804 805 add x17, x12, x8 // Calculate subroutine to jump to. 806 br x17 807 808.LisOther: // Everything else takes one vReg. 809 cmp x8, # 6*12 // Skip this load if all registers full. 810 beq .Ladvance4 811 812 add x17, x11, x8 // Calculate subroutine to jump to. 813 br x17 814 815.Ladvance4: 816 add x9, x9, #4 817 b .LfillRegisters 818 819.Ladvance8: 820 add x9, x9, #8 821 b .LfillRegisters 822 823// Macro for loading a parameter into a register. 824// counter - the register with offset into these tables 825// size - the size of the register - 4 or 8 bytes. 826// register - the name of the register to be loaded. 827.macro LOADREG counter size register return 828 ldr \register , [x9], #\size 829 add \counter, \counter, 12 830 b \return 831.endm 832 833// Store ints. 834.LstoreW2: 835 LOADREG x8 4 w2 .LfillRegisters 836 LOADREG x8 4 w3 .LfillRegisters 837 LOADREG x8 4 w4 .LfillRegisters 838 LOADREG x8 4 w5 .LfillRegisters 839 LOADREG x8 4 w6 .LfillRegisters 840 LOADREG x8 4 w7 .LfillRegisters 841 842// Store longs. 843.LstoreX2: 844 LOADREG x8 8 x2 .LfillRegisters 845 LOADREG x8 8 x3 .LfillRegisters 846 LOADREG x8 8 x4 .LfillRegisters 847 LOADREG x8 8 x5 .LfillRegisters 848 LOADREG x8 8 x6 .LfillRegisters 849 LOADREG x8 8 x7 .LfillRegisters 850 851// Store singles. 852.LstoreS0: 853 LOADREG x15 4 s0 .LfillRegisters 854 LOADREG x15 4 s1 .LfillRegisters 855 LOADREG x15 4 s2 .LfillRegisters 856 LOADREG x15 4 s3 .LfillRegisters 857 LOADREG x15 4 s4 .LfillRegisters 858 LOADREG x15 4 s5 .LfillRegisters 859 LOADREG x15 4 s6 .LfillRegisters 860 LOADREG x15 4 s7 .LfillRegisters 861 862// Store doubles. 863.LstoreD0: 864 LOADREG x15 8 d0 .LfillRegisters 865 LOADREG x15 8 d1 .LfillRegisters 866 LOADREG x15 8 d2 .LfillRegisters 867 LOADREG x15 8 d3 .LfillRegisters 868 LOADREG x15 8 d4 .LfillRegisters 869 LOADREG x15 8 d5 .LfillRegisters 870 LOADREG x15 8 d6 .LfillRegisters 871 LOADREG x15 8 d7 .LfillRegisters 872 873 874.LcallFunction: 875 876 INVOKE_STUB_CALL_AND_RETURN 877 878END art_quick_invoke_stub 879 880/* extern"C" 881 * void art_quick_invoke_static_stub(ArtMethod *method, x0 882 * uint32_t *args, x1 883 * uint32_t argsize, w2 884 * Thread *self, x3 885 * JValue *result, x4 886 * char *shorty); x5 887 */ 888ENTRY art_quick_invoke_static_stub 889 // Spill registers as per AACPS64 calling convention. 890 INVOKE_STUB_CREATE_FRAME 891 892 // Fill registers x/w1 to x/w7 and s/d0 to s/d7 with parameters. 893 // Parse the passed shorty to determine which register to load. 894 // Load addresses for routines that load WXSD registers. 895 adr x11, .LstoreW1_2 896 adr x12, .LstoreX1_2 897 adr x13, .LstoreS0_2 898 adr x14, .LstoreD0_2 899 900 // Initialize routine offsets to 0 for integers and floats. 901 // x8 for integers, x15 for floating point. 902 mov x8, #0 903 mov x15, #0 904 905 add x10, x5, #1 // Load shorty address, plus one to skip return value. 906 907 // Loop to fill registers. 908.LfillRegisters2: 909 ldrb w17, [x10], #1 // Load next character in signature, and increment. 910 cbz w17, .LcallFunction2 // Exit at end of signature. Shorty 0 terminated. 911 912 cmp w17, #'F' // is this a float? 913 bne .LisDouble2 914 915 cmp x15, # 8*12 // Skip this load if all registers full. 916 beq .Ladvance4_2 917 918 add x17, x13, x15 // Calculate subroutine to jump to. 919 br x17 920 921.LisDouble2: 922 cmp w17, #'D' // is this a double? 923 bne .LisLong2 924 925 cmp x15, # 8*12 // Skip this load if all registers full. 926 beq .Ladvance8_2 927 928 add x17, x14, x15 // Calculate subroutine to jump to. 929 br x17 930 931.LisLong2: 932 cmp w17, #'J' // is this a long? 933 bne .LisOther2 934 935 cmp x8, # 7*12 // Skip this load if all registers full. 936 beq .Ladvance8_2 937 938 add x17, x12, x8 // Calculate subroutine to jump to. 939 br x17 940 941.LisOther2: // Everything else takes one vReg. 942 cmp x8, # 7*12 // Skip this load if all registers full. 943 beq .Ladvance4_2 944 945 add x17, x11, x8 // Calculate subroutine to jump to. 946 br x17 947 948.Ladvance4_2: 949 add x9, x9, #4 950 b .LfillRegisters2 951 952.Ladvance8_2: 953 add x9, x9, #8 954 b .LfillRegisters2 955 956// Store ints. 957.LstoreW1_2: 958 LOADREG x8 4 w1 .LfillRegisters2 959 LOADREG x8 4 w2 .LfillRegisters2 960 LOADREG x8 4 w3 .LfillRegisters2 961 LOADREG x8 4 w4 .LfillRegisters2 962 LOADREG x8 4 w5 .LfillRegisters2 963 LOADREG x8 4 w6 .LfillRegisters2 964 LOADREG x8 4 w7 .LfillRegisters2 965 966// Store longs. 967.LstoreX1_2: 968 LOADREG x8 8 x1 .LfillRegisters2 969 LOADREG x8 8 x2 .LfillRegisters2 970 LOADREG x8 8 x3 .LfillRegisters2 971 LOADREG x8 8 x4 .LfillRegisters2 972 LOADREG x8 8 x5 .LfillRegisters2 973 LOADREG x8 8 x6 .LfillRegisters2 974 LOADREG x8 8 x7 .LfillRegisters2 975 976// Store singles. 977.LstoreS0_2: 978 LOADREG x15 4 s0 .LfillRegisters2 979 LOADREG x15 4 s1 .LfillRegisters2 980 LOADREG x15 4 s2 .LfillRegisters2 981 LOADREG x15 4 s3 .LfillRegisters2 982 LOADREG x15 4 s4 .LfillRegisters2 983 LOADREG x15 4 s5 .LfillRegisters2 984 LOADREG x15 4 s6 .LfillRegisters2 985 LOADREG x15 4 s7 .LfillRegisters2 986 987// Store doubles. 988.LstoreD0_2: 989 LOADREG x15 8 d0 .LfillRegisters2 990 LOADREG x15 8 d1 .LfillRegisters2 991 LOADREG x15 8 d2 .LfillRegisters2 992 LOADREG x15 8 d3 .LfillRegisters2 993 LOADREG x15 8 d4 .LfillRegisters2 994 LOADREG x15 8 d5 .LfillRegisters2 995 LOADREG x15 8 d6 .LfillRegisters2 996 LOADREG x15 8 d7 .LfillRegisters2 997 998 999.LcallFunction2: 1000 1001 INVOKE_STUB_CALL_AND_RETURN 1002 1003END art_quick_invoke_static_stub 1004 1005 1006 1007/* extern"C" void art_quick_osr_stub(void** stack, x0 1008 * size_t stack_size_in_bytes, x1 1009 * const uin8_t* native_pc, x2 1010 * JValue *result, x3 1011 * char *shorty, x4 1012 * Thread *self) x5 1013 */ 1014ENTRY art_quick_osr_stub 1015SAVE_SIZE=15*8 // x3, x4, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, SP, LR, FP saved. 1016 mov x9, sp // Save stack pointer. 1017 .cfi_register sp,x9 1018 1019 sub x10, sp, # SAVE_SIZE 1020 and x10, x10, # ~0xf // Enforce 16 byte stack alignment. 1021 mov sp, x10 // Set new SP. 1022 1023 str x28, [sp, #112] 1024 stp x26, x27, [sp, #96] 1025 stp x24, x25, [sp, #80] 1026 stp x22, x23, [sp, #64] 1027 stp x20, x21, [sp, #48] 1028 stp x9, x19, [sp, #32] // Save old stack pointer and x19. 1029 stp x3, x4, [sp, #16] // Save result and shorty addresses. 1030 stp xFP, xLR, [sp] // Store LR & FP. 1031 mov xSELF, x5 // Move thread pointer into SELF register. 1032 1033 sub sp, sp, #16 1034 str xzr, [sp] // Store null for ArtMethod* slot 1035 // Branch to stub. 1036 bl .Losr_entry 1037 add sp, sp, #16 1038 1039 // Restore return value address and shorty address. 1040 ldp x3,x4, [sp, #16] 1041 ldr x28, [sp, #112] 1042 ldp x26, x27, [sp, #96] 1043 ldp x24, x25, [sp, #80] 1044 ldp x22, x23, [sp, #64] 1045 ldp x20, x21, [sp, #48] 1046 1047 // Store result (w0/x0/s0/d0) appropriately, depending on resultType. 1048 ldrb w10, [x4] 1049 1050 // Check the return type and store the correct register into the jvalue in memory. 1051 1052 // Don't set anything for a void type. 1053 cmp w10, #'V' 1054 beq .Losr_exit 1055 1056 // Is it a double? 1057 cmp w10, #'D' 1058 bne .Lno_double 1059 str d0, [x3] 1060 b .Losr_exit 1061 1062.Lno_double: // Is it a float? 1063 cmp w10, #'F' 1064 bne .Lno_float 1065 str s0, [x3] 1066 b .Losr_exit 1067 1068.Lno_float: // Just store x0. Doesn't matter if it is 64 or 32 bits. 1069 str x0, [x3] 1070 1071.Losr_exit: // Finish up. 1072 ldp x2, x19, [sp, #32] // Restore stack pointer and x19. 1073 ldp xFP, xLR, [sp] // Restore old frame pointer and link register. 1074 mov sp, x2 1075 ret 1076 1077.Losr_entry: 1078 // Update stack pointer for the callee 1079 sub sp, sp, x1 1080 1081 // Update link register slot expected by the callee. 1082 sub w1, w1, #8 1083 str lr, [sp, x1] 1084 1085 // Copy arguments into stack frame. 1086 // Use simple copy routine for now. 1087 // 4 bytes per slot. 1088 // X0 - source address 1089 // W1 - args length 1090 // SP - destination address. 1091 // W10 - temporary 1092.Losr_loop_entry: 1093 cmp w1, #0 1094 beq .Losr_loop_exit 1095 sub w1, w1, #4 1096 ldr w10, [x0, x1] 1097 str w10, [sp, x1] 1098 b .Losr_loop_entry 1099 1100.Losr_loop_exit: 1101 // Branch to the OSR entry point. 1102 br x2 1103 1104END art_quick_osr_stub 1105 1106 /* 1107 * On entry x0 is uintptr_t* gprs_ and x1 is uint64_t* fprs_ 1108 */ 1109 1110ENTRY art_quick_do_long_jump 1111 // Load FPRs 1112 ldp d0, d1, [x1], #16 1113 ldp d2, d3, [x1], #16 1114 ldp d4, d5, [x1], #16 1115 ldp d6, d7, [x1], #16 1116 ldp d8, d9, [x1], #16 1117 ldp d10, d11, [x1], #16 1118 ldp d12, d13, [x1], #16 1119 ldp d14, d15, [x1], #16 1120 ldp d16, d17, [x1], #16 1121 ldp d18, d19, [x1], #16 1122 ldp d20, d21, [x1], #16 1123 ldp d22, d23, [x1], #16 1124 ldp d24, d25, [x1], #16 1125 ldp d26, d27, [x1], #16 1126 ldp d28, d29, [x1], #16 1127 ldp d30, d31, [x1] 1128 1129 // Load GPRs 1130 // TODO: lots of those are smashed, could optimize. 1131 add x0, x0, #30*8 1132 ldp x30, x1, [x0], #-16 // LR & SP 1133 ldp x28, x29, [x0], #-16 1134 ldp x26, x27, [x0], #-16 1135 ldp x24, x25, [x0], #-16 1136 ldp x22, x23, [x0], #-16 1137 ldp x20, x21, [x0], #-16 1138 ldp x18, x19, [x0], #-16 1139 ldp x16, x17, [x0], #-16 1140 ldp x14, x15, [x0], #-16 1141 ldp x12, x13, [x0], #-16 1142 ldp x10, x11, [x0], #-16 1143 ldp x8, x9, [x0], #-16 1144 ldp x6, x7, [x0], #-16 1145 ldp x4, x5, [x0], #-16 1146 ldp x2, x3, [x0], #-16 1147 mov sp, x1 1148 1149 // Need to load PC, it's at the end (after the space for the unused XZR). Use x1. 1150 ldr x1, [x0, #33*8] 1151 // And the value of x0. 1152 ldr x0, [x0] 1153 1154 br x1 1155END art_quick_do_long_jump 1156 1157 /* 1158 * Entry from managed code that calls artLockObjectFromCode, may block for GC. x0 holds the 1159 * possibly null object to lock. 1160 * 1161 * Derived from arm32 code. 1162 */ 1163 .extern artLockObjectFromCode 1164ENTRY art_quick_lock_object 1165 cbz w0, .Lslow_lock 1166 add x4, x0, #MIRROR_OBJECT_LOCK_WORD_OFFSET // exclusive load/store has no immediate anymore 1167.Lretry_lock: 1168 ldr w2, [xSELF, #THREAD_ID_OFFSET] // TODO: Can the thread ID really change during the loop? 1169 ldxr w1, [x4] 1170 mov x3, x1 1171 and w3, w3, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED // zero the gc bits 1172 cbnz w3, .Lnot_unlocked // already thin locked 1173 // unlocked case - x1: original lock word that's zero except for the read barrier bits. 1174 orr x2, x1, x2 // x2 holds thread id with count of 0 with preserved read barrier bits 1175 stxr w3, w2, [x4] 1176 cbnz w3, .Llock_stxr_fail // store failed, retry 1177 dmb ishld // full (LoadLoad|LoadStore) memory barrier 1178 ret 1179.Lnot_unlocked: // x1: original lock word 1180 lsr w3, w1, LOCK_WORD_STATE_SHIFT 1181 cbnz w3, .Lslow_lock // if either of the top two bits are set, go slow path 1182 eor w2, w1, w2 // lock_word.ThreadId() ^ self->ThreadId() 1183 uxth w2, w2 // zero top 16 bits 1184 cbnz w2, .Lslow_lock // lock word and self thread id's match -> recursive lock 1185 // else contention, go to slow path 1186 mov x3, x1 // copy the lock word to check count overflow. 1187 and w3, w3, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED // zero the gc bits. 1188 add w2, w3, #LOCK_WORD_THIN_LOCK_COUNT_ONE // increment count in lock word placing in w2 to check overflow 1189 lsr w3, w2, #LOCK_WORD_GC_STATE_SHIFT // if the first gc state bit is set, we overflowed. 1190 cbnz w3, .Lslow_lock // if we overflow the count go slow path 1191 add w2, w1, #LOCK_WORD_THIN_LOCK_COUNT_ONE // increment count for real 1192 stxr w3, w2, [x4] 1193 cbnz w3, .Llock_stxr_fail // store failed, retry 1194 ret 1195.Llock_stxr_fail: 1196 b .Lretry_lock // retry 1197.Lslow_lock: 1198 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case we block 1199 mov x1, xSELF // pass Thread::Current 1200 bl artLockObjectFromCode // (Object* obj, Thread*) 1201 RESTORE_SAVE_REFS_ONLY_FRAME 1202 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1203END art_quick_lock_object 1204 1205ENTRY art_quick_lock_object_no_inline 1206 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case we block 1207 mov x1, xSELF // pass Thread::Current 1208 bl artLockObjectFromCode // (Object* obj, Thread*) 1209 RESTORE_SAVE_REFS_ONLY_FRAME 1210 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1211END art_quick_lock_object_no_inline 1212 1213 /* 1214 * Entry from managed code that calls artUnlockObjectFromCode and delivers exception on failure. 1215 * x0 holds the possibly null object to lock. 1216 * 1217 * Derived from arm32 code. 1218 */ 1219 .extern artUnlockObjectFromCode 1220ENTRY art_quick_unlock_object 1221 cbz x0, .Lslow_unlock 1222 add x4, x0, #MIRROR_OBJECT_LOCK_WORD_OFFSET // exclusive load/store has no immediate anymore 1223.Lretry_unlock: 1224#ifndef USE_READ_BARRIER 1225 ldr w1, [x4] 1226#else 1227 ldxr w1, [x4] // Need to use atomic instructions for read barrier 1228#endif 1229 lsr w2, w1, LOCK_WORD_STATE_SHIFT 1230 cbnz w2, .Lslow_unlock // if either of the top two bits are set, go slow path 1231 ldr w2, [xSELF, #THREAD_ID_OFFSET] 1232 mov x3, x1 // copy lock word to check thread id equality 1233 and w3, w3, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED // zero the gc bits 1234 eor w3, w3, w2 // lock_word.ThreadId() ^ self->ThreadId() 1235 uxth w3, w3 // zero top 16 bits 1236 cbnz w3, .Lslow_unlock // do lock word and self thread id's match? 1237 mov x3, x1 // copy lock word to detect transition to unlocked 1238 and w3, w3, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED // zero the gc bits 1239 cmp w3, #LOCK_WORD_THIN_LOCK_COUNT_ONE 1240 bpl .Lrecursive_thin_unlock 1241 // transition to unlocked 1242 mov x3, x1 1243 and w3, w3, #LOCK_WORD_GC_STATE_MASK_SHIFTED // w3: zero except for the preserved read barrier bits 1244 dmb ish // full (LoadStore|StoreStore) memory barrier 1245#ifndef USE_READ_BARRIER 1246 str w3, [x4] 1247#else 1248 stxr w2, w3, [x4] // Need to use atomic instructions for read barrier 1249 cbnz w2, .Lunlock_stxr_fail // store failed, retry 1250#endif 1251 ret 1252.Lrecursive_thin_unlock: // w1: original lock word 1253 sub w1, w1, #LOCK_WORD_THIN_LOCK_COUNT_ONE // decrement count 1254#ifndef USE_READ_BARRIER 1255 str w1, [x4] 1256#else 1257 stxr w2, w1, [x4] // Need to use atomic instructions for read barrier 1258 cbnz w2, .Lunlock_stxr_fail // store failed, retry 1259#endif 1260 ret 1261.Lunlock_stxr_fail: 1262 b .Lretry_unlock // retry 1263.Lslow_unlock: 1264 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case exception allocation triggers GC 1265 mov x1, xSELF // pass Thread::Current 1266 bl artUnlockObjectFromCode // (Object* obj, Thread*) 1267 RESTORE_SAVE_REFS_ONLY_FRAME 1268 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1269END art_quick_unlock_object 1270 1271ENTRY art_quick_unlock_object_no_inline 1272 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case exception allocation triggers GC 1273 mov x1, xSELF // pass Thread::Current 1274 bl artUnlockObjectFromCode // (Object* obj, Thread*) 1275 RESTORE_SAVE_REFS_ONLY_FRAME 1276 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1277END art_quick_unlock_object_no_inline 1278 1279 /* 1280 * Entry from managed code that calls artIsAssignableFromCode and on failure calls 1281 * artThrowClassCastException. 1282 */ 1283 .extern artThrowClassCastException 1284ENTRY art_quick_check_cast 1285 // Store arguments and link register 1286 // Stack needs to be 16B aligned on calls. 1287 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 32 1288 SAVE_REG xLR, 24 1289 1290 // Call runtime code 1291 bl artIsAssignableFromCode 1292 1293 // Check for exception 1294 cbz x0, .Lthrow_class_cast_exception 1295 1296 // Restore and return 1297 .cfi_remember_state 1298 RESTORE_REG xLR, 24 1299 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 1300 ret 1301 .cfi_restore_state // Reset unwind info so following code unwinds. 1302 1303.Lthrow_class_cast_exception: 1304 // Restore 1305 RESTORE_REG xLR, 24 1306 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 1307 1308 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context 1309 mov x2, xSELF // pass Thread::Current 1310 b artThrowClassCastException // (Class*, Class*, Thread*) 1311 brk 0 // We should not return here... 1312END art_quick_check_cast 1313 1314// Restore xReg's value from [sp, #offset] if xReg is not the same as xExclude. 1315.macro POP_REG_NE xReg, offset, xExclude 1316 .ifnc \xReg, \xExclude 1317 ldr \xReg, [sp, #\offset] // restore xReg 1318 .cfi_restore \xReg 1319 .endif 1320.endm 1321 1322// Restore xReg1's value from [sp, #offset] if xReg1 is not the same as xExclude. 1323// Restore xReg2's value from [sp, #(offset + 8)] if xReg2 is not the same as xExclude. 1324.macro POP_REGS_NE xReg1, xReg2, offset, xExclude 1325 .ifc \xReg1, \xExclude 1326 ldr \xReg2, [sp, #(\offset + 8)] // restore xReg2 1327 .else 1328 .ifc \xReg2, \xExclude 1329 ldr \xReg1, [sp, #\offset] // restore xReg1 1330 .else 1331 ldp \xReg1, \xReg2, [sp, #\offset] // restore xReg1 and xReg2 1332 .endif 1333 .endif 1334 .cfi_restore \xReg1 1335 .cfi_restore \xReg2 1336.endm 1337 1338 /* 1339 * Macro to insert read barrier, only used in art_quick_aput_obj. 1340 * xDest, wDest and xObj are registers, offset is a defined literal such as 1341 * MIRROR_OBJECT_CLASS_OFFSET. Dest needs both x and w versions of the same register to handle 1342 * name mismatch between instructions. This macro uses the lower 32b of register when possible. 1343 * TODO: When read barrier has a fast path, add heap unpoisoning support for the fast path. 1344 */ 1345.macro READ_BARRIER xDest, wDest, xObj, xTemp, wTemp, offset, number 1346#ifdef USE_READ_BARRIER 1347#ifdef USE_BAKER_READ_BARRIER 1348 ldr \wTemp, [\xObj, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 1349 tbnz \wTemp, #LOCK_WORD_READ_BARRIER_STATE_SHIFT, .Lrb_slowpath\number 1350 // False dependency to avoid needing load/load fence. 1351 add \xObj, \xObj, \xTemp, lsr #32 1352 ldr \wDest, [\xObj, #\offset] // Heap reference = 32b. This also zero-extends to \xDest. 1353 UNPOISON_HEAP_REF \wDest 1354 b .Lrb_exit\number 1355#endif 1356.Lrb_slowpath\number: 1357 // Store registers used in art_quick_aput_obj (x0-x4, LR), stack is 16B aligned. 1358 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 48 1359 SAVE_TWO_REGS x2, x3, 16 1360 SAVE_TWO_REGS x4, xLR, 32 1361 1362 // mov x0, \xRef // pass ref in x0 (no-op for now since parameter ref is unused) 1363 .ifnc \xObj, x1 1364 mov x1, \xObj // pass xObj 1365 .endif 1366 mov w2, #\offset // pass offset 1367 bl artReadBarrierSlow // artReadBarrierSlow(ref, xObj, offset) 1368 // No need to unpoison return value in w0, artReadBarrierSlow() would do the unpoisoning. 1369 .ifnc \wDest, w0 1370 mov \wDest, w0 // save return value in wDest 1371 .endif 1372 1373 // Conditionally restore saved registers 1374 POP_REG_NE x0, 0, \xDest 1375 POP_REG_NE x1, 8, \xDest 1376 POP_REG_NE x2, 16, \xDest 1377 POP_REG_NE x3, 24, \xDest 1378 POP_REG_NE x4, 32, \xDest 1379 RESTORE_REG xLR, 40 1380 DECREASE_FRAME 48 1381.Lrb_exit\number: 1382#else 1383 ldr \wDest, [\xObj, #\offset] // Heap reference = 32b. This also zero-extends to \xDest. 1384 UNPOISON_HEAP_REF \wDest 1385#endif // USE_READ_BARRIER 1386.endm 1387 1388 /* 1389 * Entry from managed code for array put operations of objects where the value being stored 1390 * needs to be checked for compatibility. 1391 * x0 = array, x1 = index, x2 = value 1392 * 1393 * Currently all values should fit into w0/w1/w2, and w1 always will as indices are 32b. We 1394 * assume, though, that the upper 32b are zeroed out. At least for x1/w1 we can do better by 1395 * using index-zero-extension in load/stores. 1396 * 1397 * Temporaries: x3, x4 1398 * TODO: x4 OK? ip seems wrong here. 1399 */ 1400ENTRY art_quick_aput_obj_with_null_and_bound_check 1401 tst x0, x0 1402 bne art_quick_aput_obj_with_bound_check 1403 b art_quick_throw_null_pointer_exception 1404END art_quick_aput_obj_with_null_and_bound_check 1405 1406ENTRY art_quick_aput_obj_with_bound_check 1407 ldr w3, [x0, #MIRROR_ARRAY_LENGTH_OFFSET] 1408 cmp w3, w1 1409 bhi art_quick_aput_obj 1410 mov x0, x1 1411 mov x1, x3 1412 b art_quick_throw_array_bounds 1413END art_quick_aput_obj_with_bound_check 1414 1415#ifdef USE_READ_BARRIER 1416 .extern artReadBarrierSlow 1417#endif 1418ENTRY art_quick_aput_obj 1419 cbz x2, .Ldo_aput_null 1420 READ_BARRIER x3, w3, x0, x3, w3, MIRROR_OBJECT_CLASS_OFFSET, 0 // Heap reference = 32b 1421 // This also zero-extends to x3 1422 READ_BARRIER x3, w3, x3, x4, w4, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, 1 // Heap reference = 32b 1423 // This also zero-extends to x3 1424 READ_BARRIER x4, w4, x2, x4, w4, MIRROR_OBJECT_CLASS_OFFSET, 2 // Heap reference = 32b 1425 // This also zero-extends to x4 1426 cmp w3, w4 // value's type == array's component type - trivial assignability 1427 bne .Lcheck_assignability 1428.Ldo_aput: 1429 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1430 // "Compress" = do nothing 1431 POISON_HEAP_REF w2 1432 str w2, [x3, x1, lsl #2] // Heap reference = 32b 1433 ldr x3, [xSELF, #THREAD_CARD_TABLE_OFFSET] 1434 lsr x0, x0, #7 1435 strb w3, [x3, x0] 1436 ret 1437.Ldo_aput_null: 1438 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1439 // "Compress" = do nothing 1440 str w2, [x3, x1, lsl #2] // Heap reference = 32b 1441 ret 1442.Lcheck_assignability: 1443 // Store arguments and link register 1444 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 32 1445 SAVE_TWO_REGS x2, xLR, 16 1446 1447 // Call runtime code 1448 mov x0, x3 // Heap reference, 32b, "uncompress" = do nothing, already zero-extended 1449 mov x1, x4 // Heap reference, 32b, "uncompress" = do nothing, already zero-extended 1450 bl artIsAssignableFromCode 1451 1452 // Check for exception 1453 cbz x0, .Lthrow_array_store_exception 1454 1455 // Restore 1456 .cfi_remember_state 1457 RESTORE_TWO_REGS x2, xLR, 16 1458 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 1459 1460 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1461 // "Compress" = do nothing 1462 POISON_HEAP_REF w2 1463 str w2, [x3, x1, lsl #2] // Heap reference = 32b 1464 ldr x3, [xSELF, #THREAD_CARD_TABLE_OFFSET] 1465 lsr x0, x0, #7 1466 strb w3, [x3, x0] 1467 ret 1468 .cfi_restore_state // Reset unwind info so following code unwinds. 1469.Lthrow_array_store_exception: 1470 RESTORE_TWO_REGS x2, xLR, 16 1471 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 1472 1473 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME 1474 mov x1, x2 // Pass value. 1475 mov x2, xSELF // Pass Thread::Current. 1476 b artThrowArrayStoreException // (Object*, Object*, Thread*). 1477 brk 0 // Unreached. 1478END art_quick_aput_obj 1479 1480// Macro to facilitate adding new allocation entrypoints. 1481.macro ONE_ARG_DOWNCALL name, entrypoint, return 1482 .extern \entrypoint 1483ENTRY \name 1484 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1485 mov x1, xSELF // pass Thread::Current 1486 bl \entrypoint // (uint32_t type_idx, Method* method, Thread*) 1487 RESTORE_SAVE_REFS_ONLY_FRAME 1488 \return 1489END \name 1490.endm 1491 1492// Macro to facilitate adding new allocation entrypoints. 1493.macro TWO_ARG_DOWNCALL name, entrypoint, return 1494 .extern \entrypoint 1495ENTRY \name 1496 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1497 mov x2, xSELF // pass Thread::Current 1498 bl \entrypoint // (uint32_t type_idx, Method* method, Thread*) 1499 RESTORE_SAVE_REFS_ONLY_FRAME 1500 \return 1501END \name 1502.endm 1503 1504// Macro to facilitate adding new allocation entrypoints. 1505.macro THREE_ARG_DOWNCALL name, entrypoint, return 1506 .extern \entrypoint 1507ENTRY \name 1508 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1509 mov x3, xSELF // pass Thread::Current 1510 bl \entrypoint 1511 RESTORE_SAVE_REFS_ONLY_FRAME 1512 \return 1513END \name 1514.endm 1515 1516// Macro to facilitate adding new allocation entrypoints. 1517.macro FOUR_ARG_DOWNCALL name, entrypoint, return 1518 .extern \entrypoint 1519ENTRY \name 1520 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1521 mov x4, xSELF // pass Thread::Current 1522 bl \entrypoint // 1523 RESTORE_SAVE_REFS_ONLY_FRAME 1524 \return 1525 DELIVER_PENDING_EXCEPTION 1526END \name 1527.endm 1528 1529// Macros taking opportunity of code similarities for downcalls with referrer. 1530.macro ONE_ARG_REF_DOWNCALL name, entrypoint, return 1531 .extern \entrypoint 1532ENTRY \name 1533 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1534 ldr x1, [sp, #FRAME_SIZE_SAVE_REFS_ONLY] // Load referrer 1535 mov x2, xSELF // pass Thread::Current 1536 bl \entrypoint // (uint32_t type_idx, Method* method, Thread*, SP) 1537 RESTORE_SAVE_REFS_ONLY_FRAME 1538 \return 1539END \name 1540.endm 1541 1542.macro TWO_ARG_REF_DOWNCALL name, entrypoint, return 1543 .extern \entrypoint 1544ENTRY \name 1545 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1546 ldr x2, [sp, #FRAME_SIZE_SAVE_REFS_ONLY] // Load referrer 1547 mov x3, xSELF // pass Thread::Current 1548 bl \entrypoint 1549 RESTORE_SAVE_REFS_ONLY_FRAME 1550 \return 1551END \name 1552.endm 1553 1554.macro THREE_ARG_REF_DOWNCALL name, entrypoint, return 1555 .extern \entrypoint 1556ENTRY \name 1557 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1558 ldr x3, [sp, #FRAME_SIZE_SAVE_REFS_ONLY] // Load referrer 1559 mov x4, xSELF // pass Thread::Current 1560 bl \entrypoint 1561 RESTORE_SAVE_REFS_ONLY_FRAME 1562 \return 1563END \name 1564.endm 1565 1566.macro RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1567 cbz w0, 1f // result zero branch over 1568 ret // return 15691: 1570 DELIVER_PENDING_EXCEPTION 1571.endm 1572 1573 /* 1574 * Entry from managed code that calls artHandleFillArrayDataFromCode and delivers exception on 1575 * failure. 1576 */ 1577TWO_ARG_REF_DOWNCALL art_quick_handle_fill_data, artHandleFillArrayDataFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1578 1579 /* 1580 * Entry from managed code when uninitialized static storage, this stub will run the class 1581 * initializer and deliver the exception on error. On success the static storage base is 1582 * returned. 1583 */ 1584ONE_ARG_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1585 1586ONE_ARG_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1587ONE_ARG_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1588 1589ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1590ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1591ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1592ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1593ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1594ONE_ARG_REF_DOWNCALL art_quick_get64_static, artGet64StaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1595ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1596 1597TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1598TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1599TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1600TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1601TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1602TWO_ARG_REF_DOWNCALL art_quick_get64_instance, artGet64InstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1603TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1604 1605TWO_ARG_REF_DOWNCALL art_quick_set8_static, artSet8StaticFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1606TWO_ARG_REF_DOWNCALL art_quick_set16_static, artSet16StaticFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1607TWO_ARG_REF_DOWNCALL art_quick_set32_static, artSet32StaticFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1608TWO_ARG_REF_DOWNCALL art_quick_set_obj_static, artSetObjStaticFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1609 1610THREE_ARG_REF_DOWNCALL art_quick_set8_instance, artSet8InstanceFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1611THREE_ARG_REF_DOWNCALL art_quick_set16_instance, artSet16InstanceFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1612THREE_ARG_REF_DOWNCALL art_quick_set32_instance, artSet32InstanceFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1613THREE_ARG_REF_DOWNCALL art_quick_set64_instance, artSet64InstanceFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1614THREE_ARG_REF_DOWNCALL art_quick_set_obj_instance, artSetObjInstanceFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1615 1616// This is separated out as the argument order is different. 1617 .extern artSet64StaticFromCode 1618ENTRY art_quick_set64_static 1619 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1620 ldr x1, [sp, #FRAME_SIZE_SAVE_REFS_ONLY] // Load referrer 1621 // x2 contains the parameter 1622 mov x3, xSELF // pass Thread::Current 1623 bl artSet64StaticFromCode 1624 RESTORE_SAVE_REFS_ONLY_FRAME 1625 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1626END art_quick_set64_static 1627 1628 /* 1629 * Entry from managed code to resolve a string, this stub will 1630 * check the dex cache for a matching string (the fast path), and if not found, 1631 * it will allocate a String and deliver an exception on error. 1632 * On success the String is returned. R0 holds the string index. 1633 */ 1634 1635ENTRY art_quick_resolve_string 1636 ldr x1, [sp] // load referrer 1637 ldr w2, [x1, #ART_METHOD_DECLARING_CLASS_OFFSET] // load declaring class 1638 ldr x1, [x2, #DECLARING_CLASS_DEX_CACHE_STRINGS_OFFSET] // load string dex cache 1639 ubfx x2, x0, #0, #STRING_DEX_CACHE_HASH_BITS // get masked string index into x2 1640 ldr x2, [x1, x2, lsl #STRING_DEX_CACHE_ELEMENT_SIZE_SHIFT] // load dex cache pair into x2 1641 cmp x0, x2, lsr #32 // compare against upper 32 bits 1642 bne .Lart_quick_resolve_string_slow_path 1643 ubfx x0, x2, #0, #32 // extract lower 32 bits into x0 1644#ifdef USE_READ_BARRIER 1645 // Most common case: GC is not marking. 1646 ldr w3, [xSELF, #THREAD_IS_GC_MARKING_OFFSET] 1647 cbnz x3, .Lart_quick_resolve_string_marking 1648#endif 1649 ret 1650 1651// Slow path case, the index did not match. 1652.Lart_quick_resolve_string_slow_path: 1653 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1654 mov x1, xSELF // pass Thread::Current 1655 bl artResolveStringFromCode // (int32_t string_idx, Thread* self) 1656 RESTORE_SAVE_REFS_ONLY_FRAME 1657 RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1658 1659// GC is marking case, need to check the mark bit. 1660.Lart_quick_resolve_string_marking: 1661 ldr x3, [x0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 1662 tbnz x3, #LOCK_WORD_MARK_BIT_SHIFT, .Lart_quick_resolve_string_no_rb 1663 // Save LR so that we can return, also x1 for alignment purposes. 1664 SAVE_TWO_REGS_INCREASE_FRAME x1, xLR, 16 // Save x1, LR. 1665 bl artReadBarrierMark // Get the marked string back. 1666 RESTORE_TWO_REGS_DECREASE_FRAME x1, xLR, 16 // Restore registers. 1667.Lart_quick_resolve_string_no_rb: 1668 ret 1669 1670END art_quick_resolve_string 1671 1672// Generate the allocation entrypoints for each allocator. 1673GENERATE_ALLOC_ENTRYPOINTS_FOR_NON_REGION_TLAB_ALLOCATORS 1674// Comment out allocators that have arm64 specific asm. 1675// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT(_region_tlab, RegionTLAB) implemented in asm 1676// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB) 1677// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_region_tlab, RegionTLAB) 1678GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_region_tlab, RegionTLAB) 1679// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY(_region_tlab, RegionTLAB) implemented in asm 1680// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_region_tlab, RegionTLAB) 1681GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_WITH_ACCESS_CHECK(_region_tlab, RegionTLAB) 1682GENERATE_ALLOC_ENTRYPOINTS_CHECK_AND_ALLOC_ARRAY(_region_tlab, RegionTLAB) 1683GENERATE_ALLOC_ENTRYPOINTS_CHECK_AND_ALLOC_ARRAY_WITH_ACCESS_CHECK(_region_tlab, RegionTLAB) 1684GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_region_tlab, RegionTLAB) 1685GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_region_tlab, RegionTLAB) 1686GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_region_tlab, RegionTLAB) 1687 1688// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT(_rosalloc, RosAlloc). 1689ENTRY art_quick_alloc_object_rosalloc 1690 // Fast path rosalloc allocation. 1691 // x0: type_idx/return value, x1: ArtMethod*, xSELF(x19): Thread::Current 1692 // x2-x7: free. 1693 ldr x2, [x1, #ART_METHOD_DEX_CACHE_TYPES_OFFSET_64] // Load dex cache resolved types array 1694 // Load the class (x2) 1695 ldr w2, [x2, x0, lsl #COMPRESSED_REFERENCE_SIZE_SHIFT] 1696 cbz x2, .Lart_quick_alloc_object_rosalloc_slow_path // Check null class 1697 ldr x3, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] // Check if the thread local 1698 // allocation stack has room. 1699 // ldp won't work due to large offset. 1700 ldr x4, [xSELF, #THREAD_LOCAL_ALLOC_STACK_END_OFFSET] 1701 cmp x3, x4 1702 bhs .Lart_quick_alloc_object_rosalloc_slow_path 1703 ldr w3, [x2, #MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET] // Load the object size (x3) 1704 cmp x3, #ROSALLOC_MAX_THREAD_LOCAL_BRACKET_SIZE // Check if the size is for a thread 1705 // local allocation. Also does the 1706 // finalizable and initialization 1707 // checks. 1708 bhs .Lart_quick_alloc_object_rosalloc_slow_path 1709 // Compute the rosalloc bracket index 1710 // from the size. Since the size is 1711 // already aligned we can combine the 1712 // two shifts together. 1713 add x4, xSELF, x3, lsr #(ROSALLOC_BRACKET_QUANTUM_SIZE_SHIFT - POINTER_SIZE_SHIFT) 1714 // Subtract pointer size since ther 1715 // are no runs for 0 byte allocations 1716 // and the size is already aligned. 1717 ldr x4, [x4, #(THREAD_ROSALLOC_RUNS_OFFSET - __SIZEOF_POINTER__)] 1718 // Load the free list head (x3). This 1719 // will be the return val. 1720 ldr x3, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)] 1721 cbz x3, .Lart_quick_alloc_object_rosalloc_slow_path 1722 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber x0 and x1. 1723 ldr x1, [x3, #ROSALLOC_SLOT_NEXT_OFFSET] // Load the next pointer of the head 1724 // and update the list head with the 1725 // next pointer. 1726 str x1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)] 1727 // Store the class pointer in the 1728 // header. This also overwrites the 1729 // next pointer. The offsets are 1730 // asserted to match. 1731#if ROSALLOC_SLOT_NEXT_OFFSET != MIRROR_OBJECT_CLASS_OFFSET 1732#error "Class pointer needs to overwrite next pointer." 1733#endif 1734 POISON_HEAP_REF w2 1735 str w2, [x3, #MIRROR_OBJECT_CLASS_OFFSET] 1736 // Fence. This is "ish" not "ishst" so 1737 // that it also ensures ordering of 1738 // the object size load with respect 1739 // to later accesses to the class 1740 // object. Alternatively we could use 1741 // "ishst" if we use load-acquire for 1742 // the class status load. 1743 // Needs to be done before pushing on 1744 // allocation since Heap::VisitObjects 1745 // relies on seeing the class pointer. 1746 // b/28790624 1747 dmb ish 1748 // Push the new object onto the thread 1749 // local allocation stack and 1750 // increment the thread local 1751 // allocation stack top. 1752 ldr x1, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] 1753 str w3, [x1], #COMPRESSED_REFERENCE_SIZE // (Increment x1 as a side effect.) 1754 str x1, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] 1755 // Decrement the size of the free list 1756 ldr w1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)] 1757 sub x1, x1, #1 1758 // TODO: consider combining this store 1759 // and the list head store above using 1760 // strd. 1761 str w1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)] 1762 1763 mov x0, x3 // Set the return value and return. 1764 ret 1765.Lart_quick_alloc_object_rosalloc_slow_path: 1766 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1767 mov x2, xSELF // pass Thread::Current 1768 bl artAllocObjectFromCodeRosAlloc // (uint32_t type_idx, Method* method, Thread*) 1769 RESTORE_SAVE_REFS_ONLY_FRAME 1770 RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1771END art_quick_alloc_object_rosalloc 1772 1773 1774// The common fast path code for art_quick_alloc_array_region_tlab. 1775.macro ALLOC_ARRAY_TLAB_FAST_PATH slowPathLabel, xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1776 // Check null class 1777 cbz \wClass, \slowPathLabel 1778 ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED \slowPathLabel, \xClass, \wClass, \xCount, \wCount, \xTemp0, \wTemp0, \xTemp1, \wTemp1, \xTemp2, \wTemp2 1779.endm 1780 1781// The common fast path code for art_quick_alloc_array_region_tlab. 1782.macro ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED slowPathLabel, xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1783 // Array classes are never finalizable or uninitialized, no need to check. 1784 ldr \wTemp0, [\xClass, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET] // Load component type 1785 UNPOISON_HEAP_REF \wTemp0 1786 ldr \wTemp0, [\xTemp0, #MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET] 1787 lsr \xTemp0, \xTemp0, #PRIMITIVE_TYPE_SIZE_SHIFT_SHIFT // Component size shift is in high 16 1788 // bits. 1789 // xCount is holding a 32 bit value, 1790 // it can not overflow. 1791 lsl \xTemp1, \xCount, \xTemp0 // Calculate data size 1792 // Add array data offset and alignment. 1793 add \xTemp1, \xTemp1, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1794#if MIRROR_LONG_ARRAY_DATA_OFFSET != MIRROR_INT_ARRAY_DATA_OFFSET + 4 1795#error Long array data offset must be 4 greater than int array data offset. 1796#endif 1797 1798 add \xTemp0, \xTemp0, #1 // Add 4 to the length only if the 1799 // component size shift is 3 1800 // (for 64 bit alignment). 1801 and \xTemp0, \xTemp0, #4 1802 add \xTemp1, \xTemp1, \xTemp0 1803 and \xTemp1, \xTemp1, #OBJECT_ALIGNMENT_MASK_TOGGLED64 // Apply alignemnt mask 1804 // (addr + 7) & ~7. The mask must 1805 // be 64 bits to keep high bits in 1806 // case of overflow. 1807 // Negative sized arrays are handled here since xCount holds a zero extended 32 bit value. 1808 // Negative ints become large 64 bit unsigned ints which will always be larger than max signed 1809 // 32 bit int. Since the max shift for arrays is 3, it can not become a negative 64 bit int. 1810 cmp \xTemp1, #MIN_LARGE_OBJECT_THRESHOLD // Possibly a large object, go slow 1811 bhs \slowPathLabel // path. 1812 1813 ldr \xTemp0, [xSELF, #THREAD_LOCAL_POS_OFFSET] // Check tlab for space, note that 1814 // we use (end - begin) to handle 1815 // negative size arrays. It is 1816 // assumed that a negative size will 1817 // always be greater unsigned than 1818 // region size. 1819 ldr \xTemp2, [xSELF, #THREAD_LOCAL_END_OFFSET] 1820 sub \xTemp2, \xTemp2, \xTemp0 1821 cmp \xTemp1, \xTemp2 1822 bhi \slowPathLabel 1823 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber x0 and x1. 1824 // Move old thread_local_pos to x0 1825 // for the return value. 1826 mov x0, \xTemp0 1827 add \xTemp0, \xTemp0, \xTemp1 1828 str \xTemp0, [xSELF, #THREAD_LOCAL_POS_OFFSET] // Store new thread_local_pos. 1829 ldr \xTemp0, [xSELF, #THREAD_LOCAL_OBJECTS_OFFSET] // Increment thread_local_objects. 1830 add \xTemp0, \xTemp0, #1 1831 str \xTemp0, [xSELF, #THREAD_LOCAL_OBJECTS_OFFSET] 1832 POISON_HEAP_REF \wClass 1833 str \wClass, [x0, #MIRROR_OBJECT_CLASS_OFFSET] // Store the class pointer. 1834 str \wCount, [x0, #MIRROR_ARRAY_LENGTH_OFFSET] // Store the array length. 1835 // Fence. 1836 dmb ishst 1837 ret 1838.endm 1839 1840// The common fast path code for art_quick_alloc_object_tlab and art_quick_alloc_object_region_tlab. 1841// 1842// x0: type_idx/return value, x1: ArtMethod*, x2: Class*, xSELF(x19): Thread::Current 1843// x3-x7: free. 1844// Need to preserve x0 and x1 to the slow path. 1845.macro ALLOC_OBJECT_TLAB_FAST_PATH slowPathLabel 1846 cbz x2, \slowPathLabel // Check null class 1847 ALLOC_OBJECT_TLAB_FAST_PATH_RESOLVED \slowPathLabel 1848.endm 1849 1850// TODO: delete ALLOC_OBJECT_TLAB_FAST_PATH_RESOLVED since it is the same as 1851// ALLOC_OBJECT_TLAB_FAST_PATH_INITIALIZED. 1852.macro ALLOC_OBJECT_TLAB_FAST_PATH_RESOLVED slowPathLabel 1853 ALLOC_OBJECT_TLAB_FAST_PATH_INITIALIZED \slowPathLabel 1854.endm 1855 1856.macro ALLOC_OBJECT_TLAB_FAST_PATH_INITIALIZED slowPathLabel 1857 ldr x4, [xSELF, #THREAD_LOCAL_POS_OFFSET] 1858 ldr x5, [xSELF, #THREAD_LOCAL_END_OFFSET] 1859 ldr w7, [x2, #MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET] // Load the object size (x7). 1860 add x6, x4, x7 // Add object size to tlab pos. 1861 cmp x6, x5 // Check if it fits, overflow works 1862 // since the tlab pos and end are 32 1863 // bit values. 1864 bhi \slowPathLabel 1865 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber x0 and x1. 1866 mov x0, x4 1867 str x6, [xSELF, #THREAD_LOCAL_POS_OFFSET] // Store new thread_local_pos. 1868 ldr x5, [xSELF, #THREAD_LOCAL_OBJECTS_OFFSET] // Increment thread_local_objects. 1869 add x5, x5, #1 1870 str x5, [xSELF, #THREAD_LOCAL_OBJECTS_OFFSET] 1871 POISON_HEAP_REF w2 1872 str w2, [x0, #MIRROR_OBJECT_CLASS_OFFSET] // Store the class pointer. 1873 // Fence. This is "ish" not "ishst" so 1874 // that the code after this allocation 1875 // site will see the right values in 1876 // the fields of the class. 1877 // Alternatively we could use "ishst" 1878 // if we use load-acquire for the 1879 // object size load.) 1880 dmb ish 1881 ret 1882.endm 1883 1884// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT(_tlab, TLAB). 1885ENTRY art_quick_alloc_object_tlab 1886 // Fast path tlab allocation. 1887 // x0: type_idx/return value, x1: ArtMethod*, xSELF(x19): Thread::Current 1888 // x2-x7: free. 1889#if defined(USE_READ_BARRIER) 1890 mvn x0, xzr // Read barrier not supported here. 1891 ret // Return -1. 1892#endif 1893 ldr x2, [x1, #ART_METHOD_DEX_CACHE_TYPES_OFFSET_64] // Load dex cache resolved types array 1894 // Load the class (x2) 1895 ldr w2, [x2, x0, lsl #COMPRESSED_REFERENCE_SIZE_SHIFT] 1896 ALLOC_OBJECT_TLAB_FAST_PATH .Lart_quick_alloc_object_tlab_slow_path 1897.Lart_quick_alloc_object_tlab_slow_path: 1898 SETUP_SAVE_REFS_ONLY_FRAME // Save callee saves in case of GC. 1899 mov x2, xSELF // Pass Thread::Current. 1900 bl artAllocObjectFromCodeTLAB // (uint32_t type_idx, Method* method, Thread*) 1901 RESTORE_SAVE_REFS_ONLY_FRAME 1902 RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1903END art_quick_alloc_object_tlab 1904 1905// The common code for art_quick_alloc_object_*region_tlab 1906.macro GENERATE_ALLOC_OBJECT_REGION_TLAB name, entrypoint, fast_path, is_resolved, read_barrier 1907ENTRY \name 1908 // Fast path region tlab allocation. 1909 // x0: type_idx/resolved class/return value, x1: ArtMethod*, xSELF(x19): Thread::Current 1910 // If is_resolved is 1 then x0 is the resolved type, otherwise it is the index. 1911 // x2-x7: free. 1912#if !defined(USE_READ_BARRIER) 1913 mvn x0, xzr // Read barrier must be enabled here. 1914 ret // Return -1. 1915#endif 1916.if \is_resolved 1917 mov x2, x0 // class is actually stored in x0 already 1918.else 1919 ldr x2, [x1, #ART_METHOD_DEX_CACHE_TYPES_OFFSET_64] // Load dex cache resolved types array 1920 // Load the class (x2) 1921 ldr w2, [x2, x0, lsl #COMPRESSED_REFERENCE_SIZE_SHIFT] 1922 // If the class is null, go slow path. The check is required to read the lock word. 1923 cbz w2, .Lslow_path\name 1924.endif 1925.if \read_barrier 1926 // Most common case: GC is not marking. 1927 ldr w3, [xSELF, #THREAD_IS_GC_MARKING_OFFSET] 1928 cbnz x3, .Lmarking\name 1929.endif 1930.Ldo_allocation\name: 1931 \fast_path .Lslow_path\name 1932.Lmarking\name: 1933.if \read_barrier 1934 // GC is marking, check the lock word of the class for the mark bit. 1935 // Class is not null, check mark bit in lock word. 1936 ldr w3, [x2, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 1937 // If the bit is not zero, do the allocation. 1938 tbnz w3, #LOCK_WORD_MARK_BIT_SHIFT, .Ldo_allocation\name 1939 // The read barrier slow path. Mark 1940 // the class. 1941 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 32 // Save registers (x0, x1, lr). 1942 SAVE_REG xLR, 24 // Align sp by 16 bytes. 1943 mov x0, x2 // Pass the class as the first param. 1944 bl artReadBarrierMark 1945 mov x2, x0 // Get the (marked) class back. 1946 RESTORE_REG xLR, 24 1947 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 // Restore registers. 1948 b .Ldo_allocation\name 1949.endif 1950.Lslow_path\name: 1951 SETUP_SAVE_REFS_ONLY_FRAME // Save callee saves in case of GC. 1952 mov x2, xSELF // Pass Thread::Current. 1953 bl \entrypoint // (uint32_t type_idx, Method* method, Thread*) 1954 RESTORE_SAVE_REFS_ONLY_FRAME 1955 RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1956END \name 1957.endm 1958 1959// Use ALLOC_OBJECT_TLAB_FAST_PATH_RESOLVED since the null check is already done in GENERATE_ALLOC_OBJECT_TLAB. 1960GENERATE_ALLOC_OBJECT_REGION_TLAB art_quick_alloc_object_region_tlab, artAllocObjectFromCodeRegionTLAB, ALLOC_OBJECT_TLAB_FAST_PATH_RESOLVED, 0, 1 1961// No read barrier for the resolved or initialized cases since the caller is responsible for the 1962// read barrier due to the to-space invariant. 1963GENERATE_ALLOC_OBJECT_REGION_TLAB art_quick_alloc_object_resolved_region_tlab, artAllocObjectFromCodeResolvedRegionTLAB, ALLOC_OBJECT_TLAB_FAST_PATH_RESOLVED, 1, 0 1964GENERATE_ALLOC_OBJECT_REGION_TLAB art_quick_alloc_object_initialized_region_tlab, artAllocObjectFromCodeInitializedRegionTLAB, ALLOC_OBJECT_TLAB_FAST_PATH_INITIALIZED, 1, 0 1965 1966// TODO: We could use this macro for the normal tlab allocator too. 1967 1968// The common code for art_quick_alloc_array_*region_tlab 1969.macro GENERATE_ALLOC_ARRAY_REGION_TLAB name, entrypoint, fast_path, is_resolved 1970ENTRY \name 1971 // Fast path array allocation for region tlab allocation. 1972 // x0: uint32_t type_idx 1973 // x1: int32_t component_count 1974 // x2: ArtMethod* method 1975 // x3-x7: free. 1976#if !defined(USE_READ_BARRIER) 1977 mvn x0, xzr // Read barrier must be enabled here. 1978 ret // Return -1. 1979#endif 1980.if \is_resolved 1981 mov x3, x0 1982 // If already resolved, class is stored in x0 1983.else 1984 ldr x3, [x2, #ART_METHOD_DEX_CACHE_TYPES_OFFSET_64] // Load dex cache resolved types array 1985 // Load the class (x2) 1986 ldr w3, [x3, x0, lsl #COMPRESSED_REFERENCE_SIZE_SHIFT] 1987.endif 1988 // Most common case: GC is not marking. 1989 ldr w4, [xSELF, #THREAD_IS_GC_MARKING_OFFSET] 1990 cbnz x4, .Lmarking\name 1991.Ldo_allocation\name: 1992 \fast_path .Lslow_path\name, x3, w3, x1, w1, x4, w4, x5, w5, x6, w6 1993.Lmarking\name: 1994 // GC is marking, check the lock word of the class for the mark bit. 1995 // If the class is null, go slow path. The check is required to read the lock word. 1996 cbz w3, .Lslow_path\name 1997 // Class is not null, check mark bit in lock word. 1998 ldr w4, [x3, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 1999 // If the bit is not zero, do the allocation. 2000 tbnz w4, #LOCK_WORD_MARK_BIT_SHIFT, .Ldo_allocation\name 2001 // The read barrier slow path. Mark 2002 // the class. 2003 stp x0, x1, [sp, #-32]! // Save registers (x0, x1, x2, lr). 2004 stp x2, xLR, [sp, #16] 2005 mov x0, x3 // Pass the class as the first param. 2006 bl artReadBarrierMark 2007 mov x3, x0 // Get the (marked) class back. 2008 ldp x2, xLR, [sp, #16] 2009 ldp x0, x1, [sp], #32 // Restore registers. 2010 b .Ldo_allocation\name 2011.Lslow_path\name: 2012 // x0: uint32_t type_idx / mirror::Class* klass (if resolved) 2013 // x1: int32_t component_count 2014 // x2: ArtMethod* method 2015 // x3: Thread* self 2016 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 2017 mov x3, xSELF // pass Thread::Current 2018 bl \entrypoint 2019 RESTORE_SAVE_REFS_ONLY_FRAME 2020 RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 2021END \name 2022.endm 2023 2024GENERATE_ALLOC_ARRAY_REGION_TLAB art_quick_alloc_array_region_tlab, artAllocArrayFromCodeRegionTLAB, ALLOC_ARRAY_TLAB_FAST_PATH, 0 2025// TODO: art_quick_alloc_array_resolved_region_tlab seems to not get called. Investigate compiler. 2026GENERATE_ALLOC_ARRAY_REGION_TLAB art_quick_alloc_array_resolved_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED, 1 2027 2028 /* 2029 * Called by managed code when the thread has been asked to suspend. 2030 */ 2031 .extern artTestSuspendFromCode 2032ENTRY art_quick_test_suspend 2033 SETUP_SAVE_EVERYTHING_FRAME // save callee saves for stack crawl 2034 mov x0, xSELF 2035 bl artTestSuspendFromCode // (Thread*) 2036 RESTORE_SAVE_EVERYTHING_FRAME 2037 ret 2038END art_quick_test_suspend 2039 2040ENTRY art_quick_implicit_suspend 2041 mov x0, xSELF 2042 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves for stack crawl 2043 bl artTestSuspendFromCode // (Thread*) 2044 RESTORE_SAVE_REFS_ONLY_FRAME_AND_RETURN 2045END art_quick_implicit_suspend 2046 2047 /* 2048 * Called by managed code that is attempting to call a method on a proxy class. On entry 2049 * x0 holds the proxy method and x1 holds the receiver; The frame size of the invoked proxy 2050 * method agrees with a ref and args callee save frame. 2051 */ 2052 .extern artQuickProxyInvokeHandler 2053ENTRY art_quick_proxy_invoke_handler 2054 SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_X0 2055 mov x2, xSELF // pass Thread::Current 2056 mov x3, sp // pass SP 2057 bl artQuickProxyInvokeHandler // (Method* proxy method, receiver, Thread*, SP) 2058 ldr x2, [xSELF, THREAD_EXCEPTION_OFFSET] 2059 cbnz x2, .Lexception_in_proxy // success if no exception is pending 2060 RESTORE_SAVE_REFS_AND_ARGS_FRAME // Restore frame 2061 fmov d0, x0 // Store result in d0 in case it was float or double 2062 ret // return on success 2063.Lexception_in_proxy: 2064 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2065 DELIVER_PENDING_EXCEPTION 2066END art_quick_proxy_invoke_handler 2067 2068 /* 2069 * Called to resolve an imt conflict. 2070 * x0 is the conflict ArtMethod. 2071 * xIP1 is a hidden argument that holds the target interface method's dex method index. 2072 * 2073 * Note that this stub writes to xIP0, xIP1, and x0. 2074 */ 2075 .extern artInvokeInterfaceTrampoline 2076ENTRY art_quick_imt_conflict_trampoline 2077 ldr xIP0, [sp, #0] // Load referrer 2078 ldr xIP0, [xIP0, #ART_METHOD_DEX_CACHE_METHODS_OFFSET_64] // Load dex cache methods array 2079 ldr xIP0, [xIP0, xIP1, lsl #POINTER_SIZE_SHIFT] // Load interface method 2080 ldr xIP1, [x0, #ART_METHOD_JNI_OFFSET_64] // Load ImtConflictTable 2081 ldr x0, [xIP1] // Load first entry in ImtConflictTable. 2082.Limt_table_iterate: 2083 cmp x0, xIP0 2084 // Branch if found. Benchmarks have shown doing a branch here is better. 2085 beq .Limt_table_found 2086 // If the entry is null, the interface method is not in the ImtConflictTable. 2087 cbz x0, .Lconflict_trampoline 2088 // Iterate over the entries of the ImtConflictTable. 2089 ldr x0, [xIP1, #(2 * __SIZEOF_POINTER__)]! 2090 b .Limt_table_iterate 2091.Limt_table_found: 2092 // We successfully hit an entry in the table. Load the target method 2093 // and jump to it. 2094 ldr x0, [xIP1, #__SIZEOF_POINTER__] 2095 ldr xIP0, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 2096 br xIP0 2097.Lconflict_trampoline: 2098 // Call the runtime stub to populate the ImtConflictTable and jump to the 2099 // resolved method. 2100 INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline 2101END art_quick_imt_conflict_trampoline 2102 2103ENTRY art_quick_resolution_trampoline 2104 SETUP_SAVE_REFS_AND_ARGS_FRAME 2105 mov x2, xSELF 2106 mov x3, sp 2107 bl artQuickResolutionTrampoline // (called, receiver, Thread*, SP) 2108 cbz x0, 1f 2109 mov xIP0, x0 // Remember returned code pointer in xIP0. 2110 ldr x0, [sp, #0] // artQuickResolutionTrampoline puts called method in *SP. 2111 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2112 br xIP0 21131: 2114 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2115 DELIVER_PENDING_EXCEPTION 2116END art_quick_resolution_trampoline 2117 2118/* 2119 * Generic JNI frame layout: 2120 * 2121 * #-------------------# 2122 * | | 2123 * | caller method... | 2124 * #-------------------# <--- SP on entry 2125 * | Return X30/LR | 2126 * | X29/FP | callee save 2127 * | X28 | callee save 2128 * | X27 | callee save 2129 * | X26 | callee save 2130 * | X25 | callee save 2131 * | X24 | callee save 2132 * | X23 | callee save 2133 * | X22 | callee save 2134 * | X21 | callee save 2135 * | X20 | callee save 2136 * | X19 | callee save 2137 * | X7 | arg7 2138 * | X6 | arg6 2139 * | X5 | arg5 2140 * | X4 | arg4 2141 * | X3 | arg3 2142 * | X2 | arg2 2143 * | X1 | arg1 2144 * | D7 | float arg 8 2145 * | D6 | float arg 7 2146 * | D5 | float arg 6 2147 * | D4 | float arg 5 2148 * | D3 | float arg 4 2149 * | D2 | float arg 3 2150 * | D1 | float arg 2 2151 * | D0 | float arg 1 2152 * | Method* | <- X0 2153 * #-------------------# 2154 * | local ref cookie | // 4B 2155 * | handle scope size | // 4B 2156 * #-------------------# 2157 * | JNI Call Stack | 2158 * #-------------------# <--- SP on native call 2159 * | | 2160 * | Stack for Regs | The trampoline assembly will pop these values 2161 * | | into registers for native call 2162 * #-------------------# 2163 * | Native code ptr | 2164 * #-------------------# 2165 * | Free scratch | 2166 * #-------------------# 2167 * | Ptr to (1) | <--- SP 2168 * #-------------------# 2169 */ 2170 /* 2171 * Called to do a generic JNI down-call 2172 */ 2173ENTRY art_quick_generic_jni_trampoline 2174 SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_X0 2175 2176 // Save SP , so we can have static CFI info. 2177 mov x28, sp 2178 .cfi_def_cfa_register x28 2179 2180 // This looks the same, but is different: this will be updated to point to the bottom 2181 // of the frame when the handle scope is inserted. 2182 mov xFP, sp 2183 2184 mov xIP0, #5120 2185 sub sp, sp, xIP0 2186 2187 // prepare for artQuickGenericJniTrampoline call 2188 // (Thread*, SP) 2189 // x0 x1 <= C calling convention 2190 // xSELF xFP <= where they are 2191 2192 mov x0, xSELF // Thread* 2193 mov x1, xFP 2194 bl artQuickGenericJniTrampoline // (Thread*, sp) 2195 2196 // The C call will have registered the complete save-frame on success. 2197 // The result of the call is: 2198 // x0: pointer to native code, 0 on error. 2199 // x1: pointer to the bottom of the used area of the alloca, can restore stack till there. 2200 2201 // Check for error = 0. 2202 cbz x0, .Lexception_in_native 2203 2204 // Release part of the alloca. 2205 mov sp, x1 2206 2207 // Save the code pointer 2208 mov xIP0, x0 2209 2210 // Load parameters from frame into registers. 2211 // TODO Check with artQuickGenericJniTrampoline. 2212 // Also, check again APPCS64 - the stack arguments are interleaved. 2213 ldp x0, x1, [sp] 2214 ldp x2, x3, [sp, #16] 2215 ldp x4, x5, [sp, #32] 2216 ldp x6, x7, [sp, #48] 2217 2218 ldp d0, d1, [sp, #64] 2219 ldp d2, d3, [sp, #80] 2220 ldp d4, d5, [sp, #96] 2221 ldp d6, d7, [sp, #112] 2222 2223 add sp, sp, #128 2224 2225 blr xIP0 // native call. 2226 2227 // result sign extension is handled in C code 2228 // prepare for artQuickGenericJniEndTrampoline call 2229 // (Thread*, result, result_f) 2230 // x0 x1 x2 <= C calling convention 2231 mov x1, x0 // Result (from saved). 2232 mov x0, xSELF // Thread register. 2233 fmov x2, d0 // d0 will contain floating point result, but needs to go into x2 2234 2235 bl artQuickGenericJniEndTrampoline 2236 2237 // Pending exceptions possible. 2238 ldr x2, [xSELF, THREAD_EXCEPTION_OFFSET] 2239 cbnz x2, .Lexception_in_native 2240 2241 // Tear down the alloca. 2242 mov sp, x28 2243 .cfi_def_cfa_register sp 2244 2245 // Tear down the callee-save frame. 2246 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2247 2248 // store into fpr, for when it's a fpr return... 2249 fmov d0, x0 2250 ret 2251 2252.Lexception_in_native: 2253 // Move to x1 then sp to please assembler. 2254 ldr x1, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 2255 mov sp, x1 2256 .cfi_def_cfa_register sp 2257 # This will create a new save-all frame, required by the runtime. 2258 DELIVER_PENDING_EXCEPTION 2259END art_quick_generic_jni_trampoline 2260 2261/* 2262 * Called to bridge from the quick to interpreter ABI. On entry the arguments match those 2263 * of a quick call: 2264 * x0 = method being called/to bridge to. 2265 * x1..x7, d0..d7 = arguments to that method. 2266 */ 2267ENTRY art_quick_to_interpreter_bridge 2268 SETUP_SAVE_REFS_AND_ARGS_FRAME // Set up frame and save arguments. 2269 2270 // x0 will contain mirror::ArtMethod* method. 2271 mov x1, xSELF // How to get Thread::Current() ??? 2272 mov x2, sp 2273 2274 // uint64_t artQuickToInterpreterBridge(mirror::ArtMethod* method, Thread* self, 2275 // mirror::ArtMethod** sp) 2276 bl artQuickToInterpreterBridge 2277 2278 RESTORE_SAVE_REFS_AND_ARGS_FRAME // TODO: no need to restore arguments in this case. 2279 2280 fmov d0, x0 2281 2282 RETURN_OR_DELIVER_PENDING_EXCEPTION 2283END art_quick_to_interpreter_bridge 2284 2285 2286// 2287// Instrumentation-related stubs 2288// 2289 .extern artInstrumentationMethodEntryFromCode 2290ENTRY art_quick_instrumentation_entry 2291 SETUP_SAVE_REFS_AND_ARGS_FRAME 2292 2293 mov x20, x0 // Preserve method reference in a callee-save. 2294 2295 mov x2, xSELF 2296 mov x3, xLR 2297 bl artInstrumentationMethodEntryFromCode // (Method*, Object*, Thread*, LR) 2298 2299 mov xIP0, x0 // x0 = result of call. 2300 mov x0, x20 // Reload method reference. 2301 2302 RESTORE_SAVE_REFS_AND_ARGS_FRAME // Note: will restore xSELF 2303 adr xLR, art_quick_instrumentation_exit 2304 br xIP0 // Tail-call method with lr set to art_quick_instrumentation_exit. 2305END art_quick_instrumentation_entry 2306 2307 .extern artInstrumentationMethodExitFromCode 2308ENTRY art_quick_instrumentation_exit 2309 mov xLR, #0 // Clobber LR for later checks. 2310 2311 SETUP_SAVE_REFS_ONLY_FRAME 2312 2313 // We need to save x0 and d0. We could use a callee-save from SETUP_REF_ONLY, but then 2314 // we would need to fully restore it. As there are a lot of callee-save registers, it seems 2315 // easier to have an extra small stack area. 2316 2317 str x0, [sp, #-16]! // Save integer result. 2318 .cfi_adjust_cfa_offset 16 2319 str d0, [sp, #8] // Save floating-point result. 2320 2321 add x1, sp, #16 // Pass SP. 2322 mov x2, x0 // Pass integer result. 2323 fmov x3, d0 // Pass floating-point result. 2324 mov x0, xSELF // Pass Thread. 2325 bl artInstrumentationMethodExitFromCode // (Thread*, SP, gpr_res, fpr_res) 2326 2327 mov xIP0, x0 // Return address from instrumentation call. 2328 mov xLR, x1 // r1 is holding link register if we're to bounce to deoptimize 2329 2330 ldr d0, [sp, #8] // Restore floating-point result. 2331 ldr x0, [sp], #16 // Restore integer result, and drop stack area. 2332 .cfi_adjust_cfa_offset 16 2333 2334 POP_SAVE_REFS_ONLY_FRAME 2335 2336 br xIP0 // Tail-call out. 2337END art_quick_instrumentation_exit 2338 2339 /* 2340 * Instrumentation has requested that we deoptimize into the interpreter. The deoptimization 2341 * will long jump to the upcall with a special exception of -1. 2342 */ 2343 .extern artDeoptimize 2344ENTRY art_quick_deoptimize 2345 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME 2346 mov x0, xSELF // Pass thread. 2347 bl artDeoptimize // artDeoptimize(Thread*) 2348 brk 0 2349END art_quick_deoptimize 2350 2351 /* 2352 * Compiled code has requested that we deoptimize into the interpreter. The deoptimization 2353 * will long jump to the upcall with a special exception of -1. 2354 */ 2355 .extern artDeoptimizeFromCompiledCode 2356ENTRY art_quick_deoptimize_from_compiled_code 2357 SETUP_SAVE_EVERYTHING_FRAME 2358 mov x0, xSELF // Pass thread. 2359 bl artDeoptimizeFromCompiledCode // artDeoptimizeFromCompiledCode(Thread*) 2360 brk 0 2361END art_quick_deoptimize_from_compiled_code 2362 2363 2364 /* 2365 * String's indexOf. 2366 * 2367 * TODO: Not very optimized. 2368 * On entry: 2369 * x0: string object (known non-null) 2370 * w1: char to match (known <= 0xFFFF) 2371 * w2: Starting offset in string data 2372 */ 2373ENTRY art_quick_indexof 2374 ldr w3, [x0, #MIRROR_STRING_COUNT_OFFSET] 2375 add x0, x0, #MIRROR_STRING_VALUE_OFFSET 2376 2377 /* Clamp start to [0..count] */ 2378 cmp w2, #0 2379 csel w2, wzr, w2, lt 2380 cmp w2, w3 2381 csel w2, w3, w2, gt 2382 2383 /* Save a copy to compute result */ 2384 mov x5, x0 2385 2386 /* Build pointer to start of data to compare and pre-bias */ 2387 add x0, x0, x2, lsl #1 2388 sub x0, x0, #2 2389 2390 /* Compute iteration count */ 2391 sub w2, w3, w2 2392 2393 /* 2394 * At this point we have: 2395 * x0: start of the data to test 2396 * w1: char to compare 2397 * w2: iteration count 2398 * x5: original start of string data 2399 */ 2400 2401 subs w2, w2, #4 2402 b.lt .Lindexof_remainder 2403 2404.Lindexof_loop4: 2405 ldrh w6, [x0, #2]! 2406 ldrh w7, [x0, #2]! 2407 ldrh wIP0, [x0, #2]! 2408 ldrh wIP1, [x0, #2]! 2409 cmp w6, w1 2410 b.eq .Lmatch_0 2411 cmp w7, w1 2412 b.eq .Lmatch_1 2413 cmp wIP0, w1 2414 b.eq .Lmatch_2 2415 cmp wIP1, w1 2416 b.eq .Lmatch_3 2417 subs w2, w2, #4 2418 b.ge .Lindexof_loop4 2419 2420.Lindexof_remainder: 2421 adds w2, w2, #4 2422 b.eq .Lindexof_nomatch 2423 2424.Lindexof_loop1: 2425 ldrh w6, [x0, #2]! 2426 cmp w6, w1 2427 b.eq .Lmatch_3 2428 subs w2, w2, #1 2429 b.ne .Lindexof_loop1 2430 2431.Lindexof_nomatch: 2432 mov x0, #-1 2433 ret 2434 2435.Lmatch_0: 2436 sub x0, x0, #6 2437 sub x0, x0, x5 2438 asr x0, x0, #1 2439 ret 2440.Lmatch_1: 2441 sub x0, x0, #4 2442 sub x0, x0, x5 2443 asr x0, x0, #1 2444 ret 2445.Lmatch_2: 2446 sub x0, x0, #2 2447 sub x0, x0, x5 2448 asr x0, x0, #1 2449 ret 2450.Lmatch_3: 2451 sub x0, x0, x5 2452 asr x0, x0, #1 2453 ret 2454END art_quick_indexof 2455 2456 /* 2457 * Create a function `name` calling the ReadBarrier::Mark routine, 2458 * getting its argument and returning its result through W register 2459 * `wreg` (corresponding to X register `xreg`), saving and restoring 2460 * all caller-save registers. 2461 * 2462 * If `wreg` is different from `w0`, the generated function follows a 2463 * non-standard runtime calling convention: 2464 * - register `wreg` is used to pass the (sole) argument of this 2465 * function (instead of W0); 2466 * - register `wreg` is used to return the result of this function 2467 * (instead of W0); 2468 * - W0 is treated like a normal (non-argument) caller-save register; 2469 * - everything else is the same as in the standard runtime calling 2470 * convention (e.g. standard callee-save registers are preserved). 2471 */ 2472.macro READ_BARRIER_MARK_REG name, wreg, xreg 2473ENTRY \name 2474 // Reference is null, no work to do at all. 2475 cbz \wreg, .Lret_rb_\name 2476 /* 2477 * Allocate 46 stack slots * 8 = 368 bytes: 2478 * - 20 slots for core registers X0-X19 2479 * - 24 slots for floating-point registers D0-D7 and D16-D31 2480 * - 1 slot for return address register XLR 2481 * - 1 padding slot for 16-byte stack alignment 2482 */ 2483 // Use wIP0 as temp and check the mark bit of the reference. wIP0 is not used by the compiler. 2484 ldr wIP0, [\xreg, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 2485 tbz wIP0, #LOCK_WORD_MARK_BIT_SHIFT, .Lslow_path_rb_\name 2486 ret 2487.Lslow_path_rb_\name: 2488 // Save all potentially live caller-save core registers. 2489 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 368 2490 SAVE_TWO_REGS x2, x3, 16 2491 SAVE_TWO_REGS x4, x5, 32 2492 SAVE_TWO_REGS x6, x7, 48 2493 SAVE_TWO_REGS x8, x9, 64 2494 SAVE_TWO_REGS x10, x11, 80 2495 SAVE_TWO_REGS x12, x13, 96 2496 SAVE_TWO_REGS x14, x15, 112 2497 SAVE_TWO_REGS x16, x17, 128 2498 SAVE_TWO_REGS x18, x19, 144 2499 // Save all potentially live caller-save floating-point registers. 2500 stp d0, d1, [sp, #160] 2501 stp d2, d3, [sp, #176] 2502 stp d4, d5, [sp, #192] 2503 stp d6, d7, [sp, #208] 2504 stp d16, d17, [sp, #224] 2505 stp d18, d19, [sp, #240] 2506 stp d20, d21, [sp, #256] 2507 stp d22, d23, [sp, #272] 2508 stp d24, d25, [sp, #288] 2509 stp d26, d27, [sp, #304] 2510 stp d28, d29, [sp, #320] 2511 stp d30, d31, [sp, #336] 2512 // Save return address. 2513 // (sp + #352 is a padding slot) 2514 SAVE_REG xLR, 360 2515 2516 .ifnc \wreg, w0 2517 mov w0, \wreg // Pass arg1 - obj from `wreg` 2518 .endif 2519 bl artReadBarrierMark // artReadBarrierMark(obj) 2520 .ifnc \wreg, w0 2521 mov \wreg, w0 // Return result into `wreg` 2522 .endif 2523 2524 // Restore core regs, except `xreg`, as `wreg` is used to return the 2525 // result of this function (simply remove it from the stack instead). 2526 POP_REGS_NE x0, x1, 0, \xreg 2527 POP_REGS_NE x2, x3, 16, \xreg 2528 POP_REGS_NE x4, x5, 32, \xreg 2529 POP_REGS_NE x6, x7, 48, \xreg 2530 POP_REGS_NE x8, x9, 64, \xreg 2531 POP_REGS_NE x10, x11, 80, \xreg 2532 POP_REGS_NE x12, x13, 96, \xreg 2533 POP_REGS_NE x14, x15, 112, \xreg 2534 POP_REGS_NE x16, x17, 128, \xreg 2535 POP_REGS_NE x18, x19, 144, \xreg 2536 // Restore floating-point registers. 2537 ldp d0, d1, [sp, #160] 2538 ldp d2, d3, [sp, #176] 2539 ldp d4, d5, [sp, #192] 2540 ldp d6, d7, [sp, #208] 2541 ldp d16, d17, [sp, #224] 2542 ldp d18, d19, [sp, #240] 2543 ldp d20, d21, [sp, #256] 2544 ldp d22, d23, [sp, #272] 2545 ldp d24, d25, [sp, #288] 2546 ldp d26, d27, [sp, #304] 2547 ldp d28, d29, [sp, #320] 2548 ldp d30, d31, [sp, #336] 2549 // Restore return address and remove padding. 2550 RESTORE_REG xLR, 360 2551 DECREASE_FRAME 368 2552.Lret_rb_\name: 2553 ret 2554END \name 2555.endm 2556 2557READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg00, w0, x0 2558READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, w1, x1 2559READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, w2, x2 2560READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, w3, x3 2561READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg04, w4, x4 2562READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, w5, x5 2563READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, w6, x6 2564READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, w7, x7 2565READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, w8, x8 2566READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg09, w9, x9 2567READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, w10, x10 2568READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, w11, x11 2569READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg12, w12, x12 2570READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg13, w13, x13 2571READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg14, w14, x14 2572READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg15, w15, x15 2573// READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg16, w16, x16 ip0 is blocked 2574READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg17, w17, x17 2575READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg18, w18, x18 2576READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg19, w19, x19 2577READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg20, w20, x20 2578READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg21, w21, x21 2579READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg22, w22, x22 2580READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg23, w23, x23 2581READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg24, w24, x24 2582READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg25, w25, x25 2583READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg26, w26, x26 2584READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg27, w27, x27 2585READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg28, w28, x28 2586READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg29, w29, x29 2587