quick_entrypoints_arm64.S revision 87f3fcbd0db352157fc59148e94647ef21b73bce
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "asm_support_arm64.S" 18 19#include "arch/quick_alloc_entrypoints.S" 20 21 22 /* 23 * Macro that sets up the callee save frame to conform with 24 * Runtime::CreateCalleeSaveMethod(kSaveAll) 25 */ 26.macro SETUP_SAVE_ALL_CALLEE_SAVE_FRAME 27 adrp xIP0, :got:_ZN3art7Runtime9instance_E 28 ldr xIP0, [xIP0, #:got_lo12:_ZN3art7Runtime9instance_E] 29 30 // Our registers aren't intermixed - just spill in order. 31 ldr xIP0, [xIP0] // xIP0 = & (art::Runtime * art::Runtime.instance_) . 32 33 // xIP0 = (ArtMethod*) Runtime.instance_.callee_save_methods[kRefAndArgs] . 34 // Loads appropriate callee-save-method. 35 ldr xIP0, [xIP0, RUNTIME_SAVE_ALL_CALLEE_SAVE_FRAME_OFFSET ] 36 37 sub sp, sp, #176 38 .cfi_adjust_cfa_offset 176 39 40 // Ugly compile-time check, but we only have the preprocessor. 41#if (FRAME_SIZE_SAVE_ALL_CALLEE_SAVE != 176) 42#error "SAVE_ALL_CALLEE_SAVE_FRAME(ARM64) size not as expected." 43#endif 44 45 // Stack alignment filler [sp, #8]. 46 // FP callee-saves. 47 stp d8, d9, [sp, #16] 48 stp d10, d11, [sp, #32] 49 stp d12, d13, [sp, #48] 50 stp d14, d15, [sp, #64] 51 52 // GP callee-saves 53 stp x19, x20, [sp, #80] 54 .cfi_rel_offset x19, 80 55 .cfi_rel_offset x20, 88 56 57 stp x21, x22, [sp, #96] 58 .cfi_rel_offset x21, 96 59 .cfi_rel_offset x22, 104 60 61 stp x23, x24, [sp, #112] 62 .cfi_rel_offset x23, 112 63 .cfi_rel_offset x24, 120 64 65 stp x25, x26, [sp, #128] 66 .cfi_rel_offset x25, 128 67 .cfi_rel_offset x26, 136 68 69 stp x27, x28, [sp, #144] 70 .cfi_rel_offset x27, 144 71 .cfi_rel_offset x28, 152 72 73 stp x29, xLR, [sp, #160] 74 .cfi_rel_offset x29, 160 75 .cfi_rel_offset x30, 168 76 77 // Store ArtMethod* Runtime::callee_save_methods_[kRefsAndArgs]. 78 str xIP0, [sp] 79 // Place sp in Thread::Current()->top_quick_frame. 80 mov xIP0, sp 81 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 82.endm 83 84 /* 85 * Macro that sets up the callee save frame to conform with 86 * Runtime::CreateCalleeSaveMethod(kRefsOnly). 87 */ 88.macro SETUP_REFS_ONLY_CALLEE_SAVE_FRAME 89 adrp xIP0, :got:_ZN3art7Runtime9instance_E 90 ldr xIP0, [xIP0, #:got_lo12:_ZN3art7Runtime9instance_E] 91 92 // Our registers aren't intermixed - just spill in order. 93 ldr xIP0, [xIP0] // xIP0 = & (art::Runtime * art::Runtime.instance_) . 94 95 // xIP0 = (ArtMethod*) Runtime.instance_.callee_save_methods[kRefOnly] . 96 // Loads appropriate callee-save-method. 97 ldr xIP0, [xIP0, RUNTIME_REFS_ONLY_CALLEE_SAVE_FRAME_OFFSET ] 98 99 sub sp, sp, #96 100 .cfi_adjust_cfa_offset 96 101 102 // Ugly compile-time check, but we only have the preprocessor. 103#if (FRAME_SIZE_REFS_ONLY_CALLEE_SAVE != 96) 104#error "REFS_ONLY_CALLEE_SAVE_FRAME(ARM64) size not as expected." 105#endif 106 107 // GP callee-saves. 108 // x20 paired with ArtMethod* - see below. 109 stp x21, x22, [sp, #16] 110 .cfi_rel_offset x21, 16 111 .cfi_rel_offset x22, 24 112 113 stp x23, x24, [sp, #32] 114 .cfi_rel_offset x23, 32 115 .cfi_rel_offset x24, 40 116 117 stp x25, x26, [sp, #48] 118 .cfi_rel_offset x25, 48 119 .cfi_rel_offset x26, 56 120 121 stp x27, x28, [sp, #64] 122 .cfi_rel_offset x27, 64 123 .cfi_rel_offset x28, 72 124 125 stp x29, xLR, [sp, #80] 126 .cfi_rel_offset x29, 80 127 .cfi_rel_offset x30, 88 128 129 // Store ArtMethod* Runtime::callee_save_methods_[kRefsOnly]. 130 stp xIP0, x20, [sp] 131 .cfi_rel_offset x20, 8 132 133 // Place sp in Thread::Current()->top_quick_frame. 134 mov xIP0, sp 135 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 136.endm 137 138// TODO: Probably no need to restore registers preserved by aapcs64. 139.macro RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 140 // Callee-saves. 141 ldr x20, [sp, #8] 142 .cfi_restore x20 143 144 ldp x21, x22, [sp, #16] 145 .cfi_restore x21 146 .cfi_restore x22 147 148 ldp x23, x24, [sp, #32] 149 .cfi_restore x23 150 .cfi_restore x24 151 152 ldp x25, x26, [sp, #48] 153 .cfi_restore x25 154 .cfi_restore x26 155 156 ldp x27, x28, [sp, #64] 157 .cfi_restore x27 158 .cfi_restore x28 159 160 ldp x29, xLR, [sp, #80] 161 .cfi_restore x29 162 .cfi_restore x30 163 164 add sp, sp, #96 165 .cfi_adjust_cfa_offset -96 166.endm 167 168.macro POP_REFS_ONLY_CALLEE_SAVE_FRAME 169 add sp, sp, #96 170 .cfi_adjust_cfa_offset - 96 171.endm 172 173.macro RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME_AND_RETURN 174 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 175 ret 176.endm 177 178 179.macro SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME_INTERNAL 180 sub sp, sp, #224 181 .cfi_adjust_cfa_offset 224 182 183 // Ugly compile-time check, but we only have the preprocessor. 184#if (FRAME_SIZE_REFS_AND_ARGS_CALLEE_SAVE != 224) 185#error "REFS_AND_ARGS_CALLEE_SAVE_FRAME(ARM64) size not as expected." 186#endif 187 188 // Stack alignment filler [sp, #8]. 189 // FP args. 190 stp d0, d1, [sp, #16] 191 stp d2, d3, [sp, #32] 192 stp d4, d5, [sp, #48] 193 stp d6, d7, [sp, #64] 194 195 // Core args. 196 stp x1, x2, [sp, #80] 197 .cfi_rel_offset x1, 80 198 .cfi_rel_offset x2, 88 199 200 stp x3, x4, [sp, #96] 201 .cfi_rel_offset x3, 96 202 .cfi_rel_offset x4, 104 203 204 stp x5, x6, [sp, #112] 205 .cfi_rel_offset x5, 112 206 .cfi_rel_offset x6, 120 207 208 // x7, Callee-saves. 209 stp x7, x20, [sp, #128] 210 .cfi_rel_offset x7, 128 211 .cfi_rel_offset x20, 136 212 213 stp x21, x22, [sp, #144] 214 .cfi_rel_offset x21, 144 215 .cfi_rel_offset x22, 152 216 217 stp x23, x24, [sp, #160] 218 .cfi_rel_offset x23, 160 219 .cfi_rel_offset x24, 168 220 221 stp x25, x26, [sp, #176] 222 .cfi_rel_offset x25, 176 223 .cfi_rel_offset x26, 184 224 225 stp x27, x28, [sp, #192] 226 .cfi_rel_offset x27, 192 227 .cfi_rel_offset x28, 200 228 229 // x29(callee-save) and LR. 230 stp x29, xLR, [sp, #208] 231 .cfi_rel_offset x29, 208 232 .cfi_rel_offset x30, 216 233 234.endm 235 236 /* 237 * Macro that sets up the callee save frame to conform with 238 * Runtime::CreateCalleeSaveMethod(kRefsAndArgs). 239 * 240 * TODO This is probably too conservative - saving FP & LR. 241 */ 242.macro SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME 243 adrp xIP0, :got:_ZN3art7Runtime9instance_E 244 ldr xIP0, [xIP0, #:got_lo12:_ZN3art7Runtime9instance_E] 245 246 // Our registers aren't intermixed - just spill in order. 247 ldr xIP0, [xIP0] // xIP0 = & (art::Runtime * art::Runtime.instance_) . 248 249 // xIP0 = (ArtMethod*) Runtime.instance_.callee_save_methods[kRefAndArgs] . 250 ldr xIP0, [xIP0, RUNTIME_REFS_AND_ARGS_CALLEE_SAVE_FRAME_OFFSET ] 251 252 SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME_INTERNAL 253 254 str xIP0, [sp] // Store ArtMethod* Runtime::callee_save_methods_[kRefsAndArgs] 255 // Place sp in Thread::Current()->top_quick_frame. 256 mov xIP0, sp 257 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 258.endm 259 260.macro SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME_WITH_METHOD_IN_X0 261 SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME_INTERNAL 262 str x0, [sp, #0] // Store ArtMethod* to bottom of stack. 263 // Place sp in Thread::Current()->top_quick_frame. 264 mov xIP0, sp 265 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 266.endm 267 268// TODO: Probably no need to restore registers preserved by aapcs64. 269.macro RESTORE_REFS_AND_ARGS_CALLEE_SAVE_FRAME 270 // FP args. 271 ldp d0, d1, [sp, #16] 272 ldp d2, d3, [sp, #32] 273 ldp d4, d5, [sp, #48] 274 ldp d6, d7, [sp, #64] 275 276 // Core args. 277 ldp x1, x2, [sp, #80] 278 .cfi_restore x1 279 .cfi_restore x2 280 281 ldp x3, x4, [sp, #96] 282 .cfi_restore x3 283 .cfi_restore x4 284 285 ldp x5, x6, [sp, #112] 286 .cfi_restore x5 287 .cfi_restore x6 288 289 // x7, Callee-saves. 290 ldp x7, x20, [sp, #128] 291 .cfi_restore x7 292 .cfi_restore x20 293 294 ldp x21, x22, [sp, #144] 295 .cfi_restore x21 296 .cfi_restore x22 297 298 ldp x23, x24, [sp, #160] 299 .cfi_restore x23 300 .cfi_restore x24 301 302 ldp x25, x26, [sp, #176] 303 .cfi_restore x25 304 .cfi_restore x26 305 306 ldp x27, x28, [sp, #192] 307 .cfi_restore x27 308 .cfi_restore x28 309 310 // x29(callee-save) and LR. 311 ldp x29, xLR, [sp, #208] 312 .cfi_restore x29 313 .cfi_restore x30 314 315 add sp, sp, #224 316 .cfi_adjust_cfa_offset -224 317.endm 318 319.macro RETURN_IF_RESULT_IS_ZERO 320 cbnz x0, 1f // result non-zero branch over 321 ret // return 3221: 323.endm 324 325.macro RETURN_IF_RESULT_IS_NON_ZERO 326 cbz x0, 1f // result zero branch over 327 ret // return 3281: 329.endm 330 331 /* 332 * Macro that set calls through to artDeliverPendingExceptionFromCode, where the pending 333 * exception is Thread::Current()->exception_ 334 */ 335.macro DELIVER_PENDING_EXCEPTION 336 SETUP_SAVE_ALL_CALLEE_SAVE_FRAME 337 mov x0, xSELF 338 339 // Point of no return. 340 b artDeliverPendingExceptionFromCode // artDeliverPendingExceptionFromCode(Thread*) 341 brk 0 // Unreached 342.endm 343 344.macro RETURN_OR_DELIVER_PENDING_EXCEPTION_REG reg 345 ldr \reg, [xSELF, # THREAD_EXCEPTION_OFFSET] // Get exception field. 346 cbnz \reg, 1f 347 ret 3481: 349 DELIVER_PENDING_EXCEPTION 350.endm 351 352.macro RETURN_OR_DELIVER_PENDING_EXCEPTION 353 RETURN_OR_DELIVER_PENDING_EXCEPTION_REG xIP0 354.endm 355 356// Same as above with x1. This is helpful in stubs that want to avoid clobbering another register. 357.macro RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 358 RETURN_OR_DELIVER_PENDING_EXCEPTION_REG x1 359.endm 360 361.macro RETURN_IF_W0_IS_ZERO_OR_DELIVER 362 cbnz w0, 1f // result non-zero branch over 363 ret // return 3641: 365 DELIVER_PENDING_EXCEPTION 366.endm 367 368.macro NO_ARG_RUNTIME_EXCEPTION c_name, cxx_name 369 .extern \cxx_name 370ENTRY \c_name 371 SETUP_SAVE_ALL_CALLEE_SAVE_FRAME // save all registers as basis for long jump context 372 mov x0, xSELF // pass Thread::Current 373 b \cxx_name // \cxx_name(Thread*) 374END \c_name 375.endm 376 377.macro ONE_ARG_RUNTIME_EXCEPTION c_name, cxx_name 378 .extern \cxx_name 379ENTRY \c_name 380 SETUP_SAVE_ALL_CALLEE_SAVE_FRAME // save all registers as basis for long jump context. 381 mov x1, xSELF // pass Thread::Current. 382 b \cxx_name // \cxx_name(arg, Thread*). 383 brk 0 384END \c_name 385.endm 386 387.macro TWO_ARG_RUNTIME_EXCEPTION c_name, cxx_name 388 .extern \cxx_name 389ENTRY \c_name 390 SETUP_SAVE_ALL_CALLEE_SAVE_FRAME // save all registers as basis for long jump context 391 mov x2, xSELF // pass Thread::Current 392 b \cxx_name // \cxx_name(arg1, arg2, Thread*) 393 brk 0 394END \c_name 395.endm 396 397 /* 398 * Called by managed code, saves callee saves and then calls artThrowException 399 * that will place a mock Method* at the bottom of the stack. Arg1 holds the exception. 400 */ 401ONE_ARG_RUNTIME_EXCEPTION art_quick_deliver_exception, artDeliverExceptionFromCode 402 403 /* 404 * Called by managed code to create and deliver a NullPointerException. 405 */ 406NO_ARG_RUNTIME_EXCEPTION art_quick_throw_null_pointer_exception, artThrowNullPointerExceptionFromCode 407 408 /* 409 * Called by managed code to create and deliver an ArithmeticException. 410 */ 411NO_ARG_RUNTIME_EXCEPTION art_quick_throw_div_zero, artThrowDivZeroFromCode 412 413 /* 414 * Called by managed code to create and deliver an ArrayIndexOutOfBoundsException. Arg1 holds 415 * index, arg2 holds limit. 416 */ 417TWO_ARG_RUNTIME_EXCEPTION art_quick_throw_array_bounds, artThrowArrayBoundsFromCode 418 419 /* 420 * Called by managed code to create and deliver a StringIndexOutOfBoundsException 421 * as if thrown from a call to String.charAt(). Arg1 holds index, arg2 holds limit. 422 */ 423TWO_ARG_RUNTIME_EXCEPTION art_quick_throw_string_bounds, artThrowStringBoundsFromCode 424 425 /* 426 * Called by managed code to create and deliver a StackOverflowError. 427 */ 428NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFromCode 429 430 /* 431 * Called by managed code to create and deliver a NoSuchMethodError. 432 */ 433ONE_ARG_RUNTIME_EXCEPTION art_quick_throw_no_such_method, artThrowNoSuchMethodFromCode 434 435 /* 436 * All generated callsites for interface invokes and invocation slow paths will load arguments 437 * as usual - except instead of loading arg0/x0 with the target Method*, arg0/x0 will contain 438 * the method_idx. This wrapper will save arg1-arg3, and call the appropriate C helper. 439 * NOTE: "this" is first visible argument of the target, and so can be found in arg1/x1. 440 * 441 * The helper will attempt to locate the target and return a 128-bit result in x0/x1 consisting 442 * of the target Method* in x0 and method->code_ in x1. 443 * 444 * If unsuccessful, the helper will return null/????. There will be a pending exception in the 445 * thread and we branch to another stub to deliver it. 446 * 447 * On success this wrapper will restore arguments and *jump* to the target, leaving the lr 448 * pointing back to the original caller. 449 * 450 * Adapted from ARM32 code. 451 * 452 * Clobbers xIP0. 453 */ 454.macro INVOKE_TRAMPOLINE_BODY cxx_name 455 .extern \cxx_name 456 SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME // save callee saves in case allocation triggers GC 457 // Helper signature is always 458 // (method_idx, *this_object, *caller_method, *self, sp) 459 460 mov x2, xSELF // pass Thread::Current 461 mov x3, sp 462 bl \cxx_name // (method_idx, this, Thread*, SP) 463 mov xIP0, x1 // save Method*->code_ 464 RESTORE_REFS_AND_ARGS_CALLEE_SAVE_FRAME 465 cbz x0, 1f // did we find the target? if not go to exception delivery 466 br xIP0 // tail call to target 4671: 468 DELIVER_PENDING_EXCEPTION 469.endm 470.macro INVOKE_TRAMPOLINE c_name, cxx_name 471ENTRY \c_name 472 INVOKE_TRAMPOLINE_BODY \cxx_name 473END \c_name 474.endm 475 476INVOKE_TRAMPOLINE art_quick_invoke_interface_trampoline_with_access_check, artInvokeInterfaceTrampolineWithAccessCheck 477 478INVOKE_TRAMPOLINE art_quick_invoke_static_trampoline_with_access_check, artInvokeStaticTrampolineWithAccessCheck 479INVOKE_TRAMPOLINE art_quick_invoke_direct_trampoline_with_access_check, artInvokeDirectTrampolineWithAccessCheck 480INVOKE_TRAMPOLINE art_quick_invoke_super_trampoline_with_access_check, artInvokeSuperTrampolineWithAccessCheck 481INVOKE_TRAMPOLINE art_quick_invoke_virtual_trampoline_with_access_check, artInvokeVirtualTrampolineWithAccessCheck 482 483 484.macro INVOKE_STUB_CREATE_FRAME 485 486SAVE_SIZE=15*8 // x4, x5, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, SP, LR, FP saved. 487SAVE_SIZE_AND_METHOD=SAVE_SIZE+8 488 489 490 mov x9, sp // Save stack pointer. 491 .cfi_register sp,x9 492 493 add x10, x2, # SAVE_SIZE_AND_METHOD // calculate size of frame. 494 sub x10, sp, x10 // Calculate SP position - saves + ArtMethod* + args 495 and x10, x10, # ~0xf // Enforce 16 byte stack alignment. 496 mov sp, x10 // Set new SP. 497 498 sub x10, x9, #SAVE_SIZE // Calculate new FP (later). Done here as we must move SP 499 .cfi_def_cfa_register x10 // before this. 500 .cfi_adjust_cfa_offset SAVE_SIZE 501 502 str x28, [x10, #112] 503 .cfi_rel_offset x28, 112 504 505 stp x26, x27, [x10, #96] 506 .cfi_rel_offset x26, 96 507 .cfi_rel_offset x27, 104 508 509 stp x24, x25, [x10, #80] 510 .cfi_rel_offset x24, 80 511 .cfi_rel_offset x25, 88 512 513 stp x22, x23, [x10, #64] 514 .cfi_rel_offset x22, 64 515 .cfi_rel_offset x23, 72 516 517 stp x20, x21, [x10, #48] 518 .cfi_rel_offset x20, 48 519 .cfi_rel_offset x21, 56 520 521 stp x9, x19, [x10, #32] // Save old stack pointer and x19. 522 .cfi_rel_offset sp, 32 523 .cfi_rel_offset x19, 40 524 525 stp x4, x5, [x10, #16] // Save result and shorty addresses. 526 .cfi_rel_offset x4, 16 527 .cfi_rel_offset x5, 24 528 529 stp xFP, xLR, [x10] // Store LR & FP. 530 .cfi_rel_offset x29, 0 531 .cfi_rel_offset x30, 8 532 533 mov xFP, x10 // Use xFP now, as it's callee-saved. 534 .cfi_def_cfa_register x29 535 mov xSELF, x3 // Move thread pointer into SELF register. 536 537 // Copy arguments into stack frame. 538 // Use simple copy routine for now. 539 // 4 bytes per slot. 540 // X1 - source address 541 // W2 - args length 542 // X9 - destination address. 543 // W10 - temporary 544 add x9, sp, #8 // Destination address is bottom of stack + null. 545 546 // Copy parameters into the stack. Use numeric label as this is a macro and Clang's assembler 547 // does not have unique-id variables. 5481: 549 cmp w2, #0 550 beq 2f 551 sub w2, w2, #4 // Need 65536 bytes of range. 552 ldr w10, [x1, x2] 553 str w10, [x9, x2] 554 555 b 1b 556 5572: 558 // Store null into ArtMethod* at bottom of frame. 559 str xzr, [sp] 560.endm 561 562.macro INVOKE_STUB_CALL_AND_RETURN 563 564 // load method-> METHOD_QUICK_CODE_OFFSET 565 ldr x9, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 566 // Branch to method. 567 blr x9 568 569 // Restore return value address and shorty address. 570 ldp x4,x5, [xFP, #16] 571 .cfi_restore x4 572 .cfi_restore x5 573 574 ldr x28, [xFP, #112] 575 .cfi_restore x28 576 577 ldp x26, x27, [xFP, #96] 578 .cfi_restore x26 579 .cfi_restore x27 580 581 ldp x24, x25, [xFP, #80] 582 .cfi_restore x24 583 .cfi_restore x25 584 585 ldp x22, x23, [xFP, #64] 586 .cfi_restore x22 587 .cfi_restore x23 588 589 ldp x20, x21, [xFP, #48] 590 .cfi_restore x20 591 .cfi_restore x21 592 593 // Store result (w0/x0/s0/d0) appropriately, depending on resultType. 594 ldrb w10, [x5] 595 596 // Check the return type and store the correct register into the jvalue in memory. 597 // Use numeric label as this is a macro and Clang's assembler does not have unique-id variables. 598 599 // Don't set anything for a void type. 600 cmp w10, #'V' 601 beq 3f 602 603 // Is it a double? 604 cmp w10, #'D' 605 bne 1f 606 str d0, [x4] 607 b 3f 608 6091: // Is it a float? 610 cmp w10, #'F' 611 bne 2f 612 str s0, [x4] 613 b 3f 614 6152: // Just store x0. Doesn't matter if it is 64 or 32 bits. 616 str x0, [x4] 617 6183: // Finish up. 619 ldp x2, x19, [xFP, #32] // Restore stack pointer and x19. 620 .cfi_restore x19 621 mov sp, x2 622 .cfi_restore sp 623 624 ldp xFP, xLR, [xFP] // Restore old frame pointer and link register. 625 .cfi_restore x29 626 .cfi_restore x30 627 628 ret 629 630.endm 631 632 633/* 634 * extern"C" void art_quick_invoke_stub(ArtMethod *method, x0 635 * uint32_t *args, x1 636 * uint32_t argsize, w2 637 * Thread *self, x3 638 * JValue *result, x4 639 * char *shorty); x5 640 * +----------------------+ 641 * | | 642 * | C/C++ frame | 643 * | LR'' | 644 * | FP'' | <- SP' 645 * +----------------------+ 646 * +----------------------+ 647 * | x28 | <- TODO: Remove callee-saves. 648 * | : | 649 * | x19 | 650 * | SP' | 651 * | X5 | 652 * | X4 | Saved registers 653 * | LR' | 654 * | FP' | <- FP 655 * +----------------------+ 656 * | uint32_t out[n-1] | 657 * | : : | Outs 658 * | uint32_t out[0] | 659 * | ArtMethod* | <- SP value=null 660 * +----------------------+ 661 * 662 * Outgoing registers: 663 * x0 - Method* 664 * x1-x7 - integer parameters. 665 * d0-d7 - Floating point parameters. 666 * xSELF = self 667 * SP = & of ArtMethod* 668 * x1 = "this" pointer. 669 * 670 */ 671ENTRY art_quick_invoke_stub 672 // Spill registers as per AACPS64 calling convention. 673 INVOKE_STUB_CREATE_FRAME 674 675 // Fill registers x/w1 to x/w7 and s/d0 to s/d7 with parameters. 676 // Parse the passed shorty to determine which register to load. 677 // Load addresses for routines that load WXSD registers. 678 adr x11, .LstoreW2 679 adr x12, .LstoreX2 680 adr x13, .LstoreS0 681 adr x14, .LstoreD0 682 683 // Initialize routine offsets to 0 for integers and floats. 684 // x8 for integers, x15 for floating point. 685 mov x8, #0 686 mov x15, #0 687 688 add x10, x5, #1 // Load shorty address, plus one to skip return value. 689 ldr w1, [x9],#4 // Load "this" parameter, and increment arg pointer. 690 691 // Loop to fill registers. 692.LfillRegisters: 693 ldrb w17, [x10], #1 // Load next character in signature, and increment. 694 cbz w17, .LcallFunction // Exit at end of signature. Shorty 0 terminated. 695 696 cmp w17, #'F' // is this a float? 697 bne .LisDouble 698 699 cmp x15, # 8*12 // Skip this load if all registers full. 700 beq .Ladvance4 701 702 add x17, x13, x15 // Calculate subroutine to jump to. 703 br x17 704 705.LisDouble: 706 cmp w17, #'D' // is this a double? 707 bne .LisLong 708 709 cmp x15, # 8*12 // Skip this load if all registers full. 710 beq .Ladvance8 711 712 add x17, x14, x15 // Calculate subroutine to jump to. 713 br x17 714 715.LisLong: 716 cmp w17, #'J' // is this a long? 717 bne .LisOther 718 719 cmp x8, # 6*12 // Skip this load if all registers full. 720 beq .Ladvance8 721 722 add x17, x12, x8 // Calculate subroutine to jump to. 723 br x17 724 725.LisOther: // Everything else takes one vReg. 726 cmp x8, # 6*12 // Skip this load if all registers full. 727 beq .Ladvance4 728 729 add x17, x11, x8 // Calculate subroutine to jump to. 730 br x17 731 732.Ladvance4: 733 add x9, x9, #4 734 b .LfillRegisters 735 736.Ladvance8: 737 add x9, x9, #8 738 b .LfillRegisters 739 740// Macro for loading a parameter into a register. 741// counter - the register with offset into these tables 742// size - the size of the register - 4 or 8 bytes. 743// register - the name of the register to be loaded. 744.macro LOADREG counter size register return 745 ldr \register , [x9], #\size 746 add \counter, \counter, 12 747 b \return 748.endm 749 750// Store ints. 751.LstoreW2: 752 LOADREG x8 4 w2 .LfillRegisters 753 LOADREG x8 4 w3 .LfillRegisters 754 LOADREG x8 4 w4 .LfillRegisters 755 LOADREG x8 4 w5 .LfillRegisters 756 LOADREG x8 4 w6 .LfillRegisters 757 LOADREG x8 4 w7 .LfillRegisters 758 759// Store longs. 760.LstoreX2: 761 LOADREG x8 8 x2 .LfillRegisters 762 LOADREG x8 8 x3 .LfillRegisters 763 LOADREG x8 8 x4 .LfillRegisters 764 LOADREG x8 8 x5 .LfillRegisters 765 LOADREG x8 8 x6 .LfillRegisters 766 LOADREG x8 8 x7 .LfillRegisters 767 768// Store singles. 769.LstoreS0: 770 LOADREG x15 4 s0 .LfillRegisters 771 LOADREG x15 4 s1 .LfillRegisters 772 LOADREG x15 4 s2 .LfillRegisters 773 LOADREG x15 4 s3 .LfillRegisters 774 LOADREG x15 4 s4 .LfillRegisters 775 LOADREG x15 4 s5 .LfillRegisters 776 LOADREG x15 4 s6 .LfillRegisters 777 LOADREG x15 4 s7 .LfillRegisters 778 779// Store doubles. 780.LstoreD0: 781 LOADREG x15 8 d0 .LfillRegisters 782 LOADREG x15 8 d1 .LfillRegisters 783 LOADREG x15 8 d2 .LfillRegisters 784 LOADREG x15 8 d3 .LfillRegisters 785 LOADREG x15 8 d4 .LfillRegisters 786 LOADREG x15 8 d5 .LfillRegisters 787 LOADREG x15 8 d6 .LfillRegisters 788 LOADREG x15 8 d7 .LfillRegisters 789 790 791.LcallFunction: 792 793 INVOKE_STUB_CALL_AND_RETURN 794 795END art_quick_invoke_stub 796 797/* extern"C" 798 * void art_quick_invoke_static_stub(ArtMethod *method, x0 799 * uint32_t *args, x1 800 * uint32_t argsize, w2 801 * Thread *self, x3 802 * JValue *result, x4 803 * char *shorty); x5 804 */ 805ENTRY art_quick_invoke_static_stub 806 // Spill registers as per AACPS64 calling convention. 807 INVOKE_STUB_CREATE_FRAME 808 809 // Fill registers x/w1 to x/w7 and s/d0 to s/d7 with parameters. 810 // Parse the passed shorty to determine which register to load. 811 // Load addresses for routines that load WXSD registers. 812 adr x11, .LstoreW1_2 813 adr x12, .LstoreX1_2 814 adr x13, .LstoreS0_2 815 adr x14, .LstoreD0_2 816 817 // Initialize routine offsets to 0 for integers and floats. 818 // x8 for integers, x15 for floating point. 819 mov x8, #0 820 mov x15, #0 821 822 add x10, x5, #1 // Load shorty address, plus one to skip return value. 823 824 // Loop to fill registers. 825.LfillRegisters2: 826 ldrb w17, [x10], #1 // Load next character in signature, and increment. 827 cbz w17, .LcallFunction2 // Exit at end of signature. Shorty 0 terminated. 828 829 cmp w17, #'F' // is this a float? 830 bne .LisDouble2 831 832 cmp x15, # 8*12 // Skip this load if all registers full. 833 beq .Ladvance4_2 834 835 add x17, x13, x15 // Calculate subroutine to jump to. 836 br x17 837 838.LisDouble2: 839 cmp w17, #'D' // is this a double? 840 bne .LisLong2 841 842 cmp x15, # 8*12 // Skip this load if all registers full. 843 beq .Ladvance8_2 844 845 add x17, x14, x15 // Calculate subroutine to jump to. 846 br x17 847 848.LisLong2: 849 cmp w17, #'J' // is this a long? 850 bne .LisOther2 851 852 cmp x8, # 7*12 // Skip this load if all registers full. 853 beq .Ladvance8_2 854 855 add x17, x12, x8 // Calculate subroutine to jump to. 856 br x17 857 858.LisOther2: // Everything else takes one vReg. 859 cmp x8, # 7*12 // Skip this load if all registers full. 860 beq .Ladvance4_2 861 862 add x17, x11, x8 // Calculate subroutine to jump to. 863 br x17 864 865.Ladvance4_2: 866 add x9, x9, #4 867 b .LfillRegisters2 868 869.Ladvance8_2: 870 add x9, x9, #8 871 b .LfillRegisters2 872 873// Store ints. 874.LstoreW1_2: 875 LOADREG x8 4 w1 .LfillRegisters2 876 LOADREG x8 4 w2 .LfillRegisters2 877 LOADREG x8 4 w3 .LfillRegisters2 878 LOADREG x8 4 w4 .LfillRegisters2 879 LOADREG x8 4 w5 .LfillRegisters2 880 LOADREG x8 4 w6 .LfillRegisters2 881 LOADREG x8 4 w7 .LfillRegisters2 882 883// Store longs. 884.LstoreX1_2: 885 LOADREG x8 8 x1 .LfillRegisters2 886 LOADREG x8 8 x2 .LfillRegisters2 887 LOADREG x8 8 x3 .LfillRegisters2 888 LOADREG x8 8 x4 .LfillRegisters2 889 LOADREG x8 8 x5 .LfillRegisters2 890 LOADREG x8 8 x6 .LfillRegisters2 891 LOADREG x8 8 x7 .LfillRegisters2 892 893// Store singles. 894.LstoreS0_2: 895 LOADREG x15 4 s0 .LfillRegisters2 896 LOADREG x15 4 s1 .LfillRegisters2 897 LOADREG x15 4 s2 .LfillRegisters2 898 LOADREG x15 4 s3 .LfillRegisters2 899 LOADREG x15 4 s4 .LfillRegisters2 900 LOADREG x15 4 s5 .LfillRegisters2 901 LOADREG x15 4 s6 .LfillRegisters2 902 LOADREG x15 4 s7 .LfillRegisters2 903 904// Store doubles. 905.LstoreD0_2: 906 LOADREG x15 8 d0 .LfillRegisters2 907 LOADREG x15 8 d1 .LfillRegisters2 908 LOADREG x15 8 d2 .LfillRegisters2 909 LOADREG x15 8 d3 .LfillRegisters2 910 LOADREG x15 8 d4 .LfillRegisters2 911 LOADREG x15 8 d5 .LfillRegisters2 912 LOADREG x15 8 d6 .LfillRegisters2 913 LOADREG x15 8 d7 .LfillRegisters2 914 915 916.LcallFunction2: 917 918 INVOKE_STUB_CALL_AND_RETURN 919 920END art_quick_invoke_static_stub 921 922 923 924/* extern"C" void art_quick_osr_stub(void** stack, x0 925 * size_t stack_size_in_bytes, x1 926 * const uin8_t* native_pc, x2 927 * JValue *result, x3 928 * char *shorty, x4 929 * Thread *self) x5 930 */ 931ENTRY art_quick_osr_stub 932SAVE_SIZE=15*8 // x3, x4, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, SP, LR, FP saved. 933 mov x9, sp // Save stack pointer. 934 .cfi_register sp,x9 935 936 sub x10, sp, # SAVE_SIZE 937 and x10, x10, # ~0xf // Enforce 16 byte stack alignment. 938 mov sp, x10 // Set new SP. 939 940 str x28, [sp, #112] 941 stp x26, x27, [sp, #96] 942 stp x24, x25, [sp, #80] 943 stp x22, x23, [sp, #64] 944 stp x20, x21, [sp, #48] 945 stp x9, x19, [sp, #32] // Save old stack pointer and x19. 946 stp x3, x4, [sp, #16] // Save result and shorty addresses. 947 stp xFP, xLR, [sp] // Store LR & FP. 948 mov xSELF, x5 // Move thread pointer into SELF register. 949 950 sub sp, sp, #16 951 str xzr, [sp] // Store null for ArtMethod* slot 952 // Branch to stub. 953 bl .Losr_entry 954 add sp, sp, #16 955 956 // Restore return value address and shorty address. 957 ldp x3,x4, [sp, #16] 958 ldr x28, [sp, #112] 959 ldp x26, x27, [sp, #96] 960 ldp x24, x25, [sp, #80] 961 ldp x22, x23, [sp, #64] 962 ldp x20, x21, [sp, #48] 963 964 // Store result (w0/x0/s0/d0) appropriately, depending on resultType. 965 ldrb w10, [x4] 966 967 // Check the return type and store the correct register into the jvalue in memory. 968 969 // Don't set anything for a void type. 970 cmp w10, #'V' 971 beq .Losr_exit 972 973 // Is it a double? 974 cmp w10, #'D' 975 bne .Lno_double 976 str d0, [x3] 977 b .Losr_exit 978 979.Lno_double: // Is it a float? 980 cmp w10, #'F' 981 bne .Lno_float 982 str s0, [x3] 983 b .Losr_exit 984 985.Lno_float: // Just store x0. Doesn't matter if it is 64 or 32 bits. 986 str x0, [x3] 987 988.Losr_exit: // Finish up. 989 ldp x2, x19, [sp, #32] // Restore stack pointer and x19. 990 ldp xFP, xLR, [sp] // Restore old frame pointer and link register. 991 mov sp, x2 992 ret 993 994.Losr_entry: 995 // Update stack pointer for the callee 996 sub sp, sp, x1 997 998 // Update link register slot expected by the callee. 999 sub w1, w1, #8 1000 str lr, [sp, x1] 1001 1002 // Copy arguments into stack frame. 1003 // Use simple copy routine for now. 1004 // 4 bytes per slot. 1005 // X0 - source address 1006 // W1 - args length 1007 // SP - destination address. 1008 // W10 - temporary 1009.Losr_loop_entry: 1010 cmp w1, #0 1011 beq .Losr_loop_exit 1012 sub w1, w1, #4 1013 ldr w10, [x0, x1] 1014 str w10, [sp, x1] 1015 b .Losr_loop_entry 1016 1017.Losr_loop_exit: 1018 // Branch to the OSR entry point. 1019 br x2 1020 1021END art_quick_osr_stub 1022 1023 /* 1024 * On entry x0 is uintptr_t* gprs_ and x1 is uint64_t* fprs_ 1025 */ 1026 1027ENTRY art_quick_do_long_jump 1028 // Load FPRs 1029 ldp d0, d1, [x1], #16 1030 ldp d2, d3, [x1], #16 1031 ldp d4, d5, [x1], #16 1032 ldp d6, d7, [x1], #16 1033 ldp d8, d9, [x1], #16 1034 ldp d10, d11, [x1], #16 1035 ldp d12, d13, [x1], #16 1036 ldp d14, d15, [x1], #16 1037 ldp d16, d17, [x1], #16 1038 ldp d18, d19, [x1], #16 1039 ldp d20, d21, [x1], #16 1040 ldp d22, d23, [x1], #16 1041 ldp d24, d25, [x1], #16 1042 ldp d26, d27, [x1], #16 1043 ldp d28, d29, [x1], #16 1044 ldp d30, d31, [x1] 1045 1046 // Load GPRs 1047 // TODO: lots of those are smashed, could optimize. 1048 add x0, x0, #30*8 1049 ldp x30, x1, [x0], #-16 // LR & SP 1050 ldp x28, x29, [x0], #-16 1051 ldp x26, x27, [x0], #-16 1052 ldp x24, x25, [x0], #-16 1053 ldp x22, x23, [x0], #-16 1054 ldp x20, x21, [x0], #-16 1055 ldp x18, x19, [x0], #-16 1056 ldp x16, x17, [x0], #-16 1057 ldp x14, x15, [x0], #-16 1058 ldp x12, x13, [x0], #-16 1059 ldp x10, x11, [x0], #-16 1060 ldp x8, x9, [x0], #-16 1061 ldp x6, x7, [x0], #-16 1062 ldp x4, x5, [x0], #-16 1063 ldp x2, x3, [x0], #-16 1064 mov sp, x1 1065 1066 // Need to load PC, it's at the end (after the space for the unused XZR). Use x1. 1067 ldr x1, [x0, #33*8] 1068 // And the value of x0. 1069 ldr x0, [x0] 1070 1071 br x1 1072END art_quick_do_long_jump 1073 1074 /* 1075 * Entry from managed code that calls artLockObjectFromCode, may block for GC. x0 holds the 1076 * possibly null object to lock. 1077 * 1078 * Derived from arm32 code. 1079 */ 1080 .extern artLockObjectFromCode 1081ENTRY art_quick_lock_object 1082 cbz w0, .Lslow_lock 1083 add x4, x0, #MIRROR_OBJECT_LOCK_WORD_OFFSET // exclusive load/store has no immediate anymore 1084.Lretry_lock: 1085 ldr w2, [xSELF, #THREAD_ID_OFFSET] // TODO: Can the thread ID really change during the loop? 1086 ldxr w1, [x4] 1087 mov x3, x1 1088 and w3, w3, #LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED // zero the read barrier bits 1089 cbnz w3, .Lnot_unlocked // already thin locked 1090 // unlocked case - x1: original lock word that's zero except for the read barrier bits. 1091 orr x2, x1, x2 // x2 holds thread id with count of 0 with preserved read barrier bits 1092 stxr w3, w2, [x4] 1093 cbnz w3, .Llock_stxr_fail // store failed, retry 1094 dmb ishld // full (LoadLoad|LoadStore) memory barrier 1095 ret 1096.Lnot_unlocked: // x1: original lock word 1097 lsr w3, w1, LOCK_WORD_STATE_SHIFT 1098 cbnz w3, .Lslow_lock // if either of the top two bits are set, go slow path 1099 eor w2, w1, w2 // lock_word.ThreadId() ^ self->ThreadId() 1100 uxth w2, w2 // zero top 16 bits 1101 cbnz w2, .Lslow_lock // lock word and self thread id's match -> recursive lock 1102 // else contention, go to slow path 1103 mov x3, x1 // copy the lock word to check count overflow. 1104 and w3, w3, #LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED // zero the read barrier bits. 1105 add w2, w3, #LOCK_WORD_THIN_LOCK_COUNT_ONE // increment count in lock word placing in w2 to check overflow 1106 lsr w3, w2, LOCK_WORD_READ_BARRIER_STATE_SHIFT // if either of the upper two bits (28-29) are set, we overflowed. 1107 cbnz w3, .Lslow_lock // if we overflow the count go slow path 1108 add w2, w1, #LOCK_WORD_THIN_LOCK_COUNT_ONE // increment count for real 1109 stxr w3, w2, [x4] 1110 cbnz w3, .Llock_stxr_fail // store failed, retry 1111 ret 1112.Llock_stxr_fail: 1113 b .Lretry_lock // retry 1114.Lslow_lock: 1115 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves in case we block 1116 mov x1, xSELF // pass Thread::Current 1117 bl artLockObjectFromCode // (Object* obj, Thread*) 1118 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1119 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1120END art_quick_lock_object 1121 1122ENTRY art_quick_lock_object_no_inline 1123 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves in case we block 1124 mov x1, xSELF // pass Thread::Current 1125 bl artLockObjectFromCode // (Object* obj, Thread*) 1126 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1127 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1128END art_quick_lock_object_no_inline 1129 1130 /* 1131 * Entry from managed code that calls artUnlockObjectFromCode and delivers exception on failure. 1132 * x0 holds the possibly null object to lock. 1133 * 1134 * Derived from arm32 code. 1135 */ 1136 .extern artUnlockObjectFromCode 1137ENTRY art_quick_unlock_object 1138 cbz x0, .Lslow_unlock 1139 add x4, x0, #MIRROR_OBJECT_LOCK_WORD_OFFSET // exclusive load/store has no immediate anymore 1140.Lretry_unlock: 1141#ifndef USE_READ_BARRIER 1142 ldr w1, [x4] 1143#else 1144 ldxr w1, [x4] // Need to use atomic instructions for read barrier 1145#endif 1146 lsr w2, w1, LOCK_WORD_STATE_SHIFT 1147 cbnz w2, .Lslow_unlock // if either of the top two bits are set, go slow path 1148 ldr w2, [xSELF, #THREAD_ID_OFFSET] 1149 mov x3, x1 // copy lock word to check thread id equality 1150 and w3, w3, #LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED // zero the read barrier bits 1151 eor w3, w3, w2 // lock_word.ThreadId() ^ self->ThreadId() 1152 uxth w3, w3 // zero top 16 bits 1153 cbnz w3, .Lslow_unlock // do lock word and self thread id's match? 1154 mov x3, x1 // copy lock word to detect transition to unlocked 1155 and w3, w3, #LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED // zero the read barrier bits 1156 cmp w3, #LOCK_WORD_THIN_LOCK_COUNT_ONE 1157 bpl .Lrecursive_thin_unlock 1158 // transition to unlocked 1159 mov x3, x1 1160 and w3, w3, #LOCK_WORD_READ_BARRIER_STATE_MASK // w3: zero except for the preserved read barrier bits 1161 dmb ish // full (LoadStore|StoreStore) memory barrier 1162#ifndef USE_READ_BARRIER 1163 str w3, [x4] 1164#else 1165 stxr w2, w3, [x4] // Need to use atomic instructions for read barrier 1166 cbnz w2, .Lunlock_stxr_fail // store failed, retry 1167#endif 1168 ret 1169.Lrecursive_thin_unlock: // w1: original lock word 1170 sub w1, w1, #LOCK_WORD_THIN_LOCK_COUNT_ONE // decrement count 1171#ifndef USE_READ_BARRIER 1172 str w1, [x4] 1173#else 1174 stxr w2, w1, [x4] // Need to use atomic instructions for read barrier 1175 cbnz w2, .Lunlock_stxr_fail // store failed, retry 1176#endif 1177 ret 1178.Lunlock_stxr_fail: 1179 b .Lretry_unlock // retry 1180.Lslow_unlock: 1181 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves in case exception allocation triggers GC 1182 mov x1, xSELF // pass Thread::Current 1183 bl artUnlockObjectFromCode // (Object* obj, Thread*) 1184 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1185 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1186END art_quick_unlock_object 1187 1188ENTRY art_quick_unlock_object_no_inline 1189 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves in case exception allocation triggers GC 1190 mov x1, xSELF // pass Thread::Current 1191 bl artUnlockObjectFromCode // (Object* obj, Thread*) 1192 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1193 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1194END art_quick_unlock_object_no_inline 1195 1196 /* 1197 * Entry from managed code that calls artIsAssignableFromCode and on failure calls 1198 * artThrowClassCastException. 1199 */ 1200 .extern artThrowClassCastException 1201ENTRY art_quick_check_cast 1202 // Store arguments and link register 1203 // Stack needs to be 16B aligned on calls. 1204 stp x0, x1, [sp,#-32]! 1205 .cfi_adjust_cfa_offset 32 1206 .cfi_rel_offset x0, 0 1207 .cfi_rel_offset x1, 8 1208 str xLR, [sp, #24] 1209 .cfi_rel_offset x30, 24 1210 1211 // Call runtime code 1212 bl artIsAssignableFromCode 1213 1214 // Check for exception 1215 cbz x0, .Lthrow_class_cast_exception 1216 1217 // Restore and return 1218 ldr xLR, [sp, #24] 1219 .cfi_restore x30 1220 ldp x0, x1, [sp], #32 1221 .cfi_restore x0 1222 .cfi_restore x1 1223 .cfi_adjust_cfa_offset -32 1224 ret 1225 1226 .cfi_adjust_cfa_offset 32 // Reset unwind info so following code unwinds. 1227 1228.Lthrow_class_cast_exception: 1229 // Restore 1230 ldr xLR, [sp, #24] 1231 .cfi_restore x30 1232 ldp x0, x1, [sp], #32 1233 .cfi_restore x0 1234 .cfi_restore x1 1235 .cfi_adjust_cfa_offset -32 1236 1237 SETUP_SAVE_ALL_CALLEE_SAVE_FRAME // save all registers as basis for long jump context 1238 mov x2, xSELF // pass Thread::Current 1239 b artThrowClassCastException // (Class*, Class*, Thread*) 1240 brk 0 // We should not return here... 1241END art_quick_check_cast 1242 1243// Restore xReg's value from [sp, #offset] if xReg is not the same as xExclude. 1244.macro POP_REG_NE xReg, offset, xExclude 1245 .ifnc \xReg, \xExclude 1246 ldr \xReg, [sp, #\offset] // restore xReg 1247 .cfi_restore \xReg 1248 .endif 1249.endm 1250 1251 /* 1252 * Macro to insert read barrier, only used in art_quick_aput_obj. 1253 * xDest, wDest and xObj are registers, offset is a defined literal such as 1254 * MIRROR_OBJECT_CLASS_OFFSET. Dest needs both x and w versions of the same register to handle 1255 * name mismatch between instructions. This macro uses the lower 32b of register when possible. 1256 * TODO: When read barrier has a fast path, add heap unpoisoning support for the fast path. 1257 */ 1258.macro READ_BARRIER xDest, wDest, xObj, offset 1259#ifdef USE_READ_BARRIER 1260 // Store registers used in art_quick_aput_obj (x0-x4, LR), stack is 16B aligned. 1261 stp x0, x1, [sp, #-48]! 1262 .cfi_adjust_cfa_offset 48 1263 .cfi_rel_offset x0, 0 1264 .cfi_rel_offset x1, 8 1265 stp x2, x3, [sp, #16] 1266 .cfi_rel_offset x2, 16 1267 .cfi_rel_offset x3, 24 1268 stp x4, xLR, [sp, #32] 1269 .cfi_rel_offset x4, 32 1270 .cfi_rel_offset x30, 40 1271 1272 // mov x0, \xRef // pass ref in x0 (no-op for now since parameter ref is unused) 1273 .ifnc \xObj, x1 1274 mov x1, \xObj // pass xObj 1275 .endif 1276 mov w2, #\offset // pass offset 1277 bl artReadBarrierSlow // artReadBarrierSlow(ref, xObj, offset) 1278 // No need to unpoison return value in w0, artReadBarrierSlow() would do the unpoisoning. 1279 .ifnc \wDest, w0 1280 mov \wDest, w0 // save return value in wDest 1281 .endif 1282 1283 // Conditionally restore saved registers 1284 POP_REG_NE x0, 0, \xDest 1285 POP_REG_NE x1, 8, \xDest 1286 POP_REG_NE x2, 16, \xDest 1287 POP_REG_NE x3, 24, \xDest 1288 POP_REG_NE x4, 32, \xDest 1289 ldr xLR, [sp, #40] 1290 .cfi_restore x30 1291 add sp, sp, #48 1292 .cfi_adjust_cfa_offset -48 1293#else 1294 ldr \wDest, [\xObj, #\offset] // Heap reference = 32b. This also zero-extends to \xDest. 1295 UNPOISON_HEAP_REF \wDest 1296#endif // USE_READ_BARRIER 1297.endm 1298 1299 /* 1300 * Entry from managed code for array put operations of objects where the value being stored 1301 * needs to be checked for compatibility. 1302 * x0 = array, x1 = index, x2 = value 1303 * 1304 * Currently all values should fit into w0/w1/w2, and w1 always will as indices are 32b. We 1305 * assume, though, that the upper 32b are zeroed out. At least for x1/w1 we can do better by 1306 * using index-zero-extension in load/stores. 1307 * 1308 * Temporaries: x3, x4 1309 * TODO: x4 OK? ip seems wrong here. 1310 */ 1311ENTRY art_quick_aput_obj_with_null_and_bound_check 1312 tst x0, x0 1313 bne art_quick_aput_obj_with_bound_check 1314 b art_quick_throw_null_pointer_exception 1315END art_quick_aput_obj_with_null_and_bound_check 1316 1317ENTRY art_quick_aput_obj_with_bound_check 1318 ldr w3, [x0, #MIRROR_ARRAY_LENGTH_OFFSET] 1319 cmp w3, w1 1320 bhi art_quick_aput_obj 1321 mov x0, x1 1322 mov x1, x3 1323 b art_quick_throw_array_bounds 1324END art_quick_aput_obj_with_bound_check 1325 1326#ifdef USE_READ_BARRIER 1327 .extern artReadBarrierSlow 1328#endif 1329ENTRY art_quick_aput_obj 1330 cbz x2, .Ldo_aput_null 1331 READ_BARRIER x3, w3, x0, MIRROR_OBJECT_CLASS_OFFSET // Heap reference = 32b 1332 // This also zero-extends to x3 1333 READ_BARRIER x4, w4, x2, MIRROR_OBJECT_CLASS_OFFSET // Heap reference = 32b 1334 // This also zero-extends to x4 1335 READ_BARRIER x3, w3, x3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET // Heap reference = 32b 1336 // This also zero-extends to x3 1337 cmp w3, w4 // value's type == array's component type - trivial assignability 1338 bne .Lcheck_assignability 1339.Ldo_aput: 1340 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1341 // "Compress" = do nothing 1342 POISON_HEAP_REF w2 1343 str w2, [x3, x1, lsl #2] // Heap reference = 32b 1344 ldr x3, [xSELF, #THREAD_CARD_TABLE_OFFSET] 1345 lsr x0, x0, #7 1346 strb w3, [x3, x0] 1347 ret 1348.Ldo_aput_null: 1349 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1350 // "Compress" = do nothing 1351 str w2, [x3, x1, lsl #2] // Heap reference = 32b 1352 ret 1353.Lcheck_assignability: 1354 // Store arguments and link register 1355 stp x0, x1, [sp,#-32]! 1356 .cfi_adjust_cfa_offset 32 1357 .cfi_rel_offset x0, 0 1358 .cfi_rel_offset x1, 8 1359 stp x2, xLR, [sp, #16] 1360 .cfi_rel_offset x2, 16 1361 .cfi_rel_offset x30, 24 1362 1363 // Call runtime code 1364 mov x0, x3 // Heap reference, 32b, "uncompress" = do nothing, already zero-extended 1365 mov x1, x4 // Heap reference, 32b, "uncompress" = do nothing, already zero-extended 1366 bl artIsAssignableFromCode 1367 1368 // Check for exception 1369 cbz x0, .Lthrow_array_store_exception 1370 1371 // Restore 1372 ldp x2, x30, [sp, #16] 1373 .cfi_restore x2 1374 .cfi_restore x30 1375 ldp x0, x1, [sp], #32 1376 .cfi_restore x0 1377 .cfi_restore x1 1378 .cfi_adjust_cfa_offset -32 1379 1380 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1381 // "Compress" = do nothing 1382 POISON_HEAP_REF w2 1383 str w2, [x3, x1, lsl #2] // Heap reference = 32b 1384 ldr x3, [xSELF, #THREAD_CARD_TABLE_OFFSET] 1385 lsr x0, x0, #7 1386 strb w3, [x3, x0] 1387 ret 1388 .cfi_adjust_cfa_offset 32 // 4 restores after cbz for unwinding. 1389.Lthrow_array_store_exception: 1390 ldp x2, x30, [sp, #16] 1391 .cfi_restore x2 1392 .cfi_restore x30 1393 ldp x0, x1, [sp], #32 1394 .cfi_restore x0 1395 .cfi_restore x1 1396 .cfi_adjust_cfa_offset -32 1397 1398 SETUP_SAVE_ALL_CALLEE_SAVE_FRAME 1399 mov x1, x2 // Pass value. 1400 mov x2, xSELF // Pass Thread::Current. 1401 b artThrowArrayStoreException // (Object*, Object*, Thread*). 1402 brk 0 // Unreached. 1403END art_quick_aput_obj 1404 1405// Macro to facilitate adding new allocation entrypoints. 1406.macro ONE_ARG_DOWNCALL name, entrypoint, return 1407 .extern \entrypoint 1408ENTRY \name 1409 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1410 mov x1, xSELF // pass Thread::Current 1411 bl \entrypoint // (uint32_t type_idx, Method* method, Thread*) 1412 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1413 \return 1414END \name 1415.endm 1416 1417// Macro to facilitate adding new allocation entrypoints. 1418.macro TWO_ARG_DOWNCALL name, entrypoint, return 1419 .extern \entrypoint 1420ENTRY \name 1421 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1422 mov x2, xSELF // pass Thread::Current 1423 bl \entrypoint // (uint32_t type_idx, Method* method, Thread*) 1424 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1425 \return 1426END \name 1427.endm 1428 1429// Macro to facilitate adding new allocation entrypoints. 1430.macro THREE_ARG_DOWNCALL name, entrypoint, return 1431 .extern \entrypoint 1432ENTRY \name 1433 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1434 mov x3, xSELF // pass Thread::Current 1435 bl \entrypoint 1436 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1437 \return 1438END \name 1439.endm 1440 1441// Macro to facilitate adding new allocation entrypoints. 1442.macro FOUR_ARG_DOWNCALL name, entrypoint, return 1443 .extern \entrypoint 1444ENTRY \name 1445 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1446 mov x4, xSELF // pass Thread::Current 1447 bl \entrypoint // 1448 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1449 \return 1450 DELIVER_PENDING_EXCEPTION 1451END \name 1452.endm 1453 1454// Macros taking opportunity of code similarities for downcalls with referrer. 1455.macro ONE_ARG_REF_DOWNCALL name, entrypoint, return 1456 .extern \entrypoint 1457ENTRY \name 1458 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1459 ldr x1, [sp, #FRAME_SIZE_REFS_ONLY_CALLEE_SAVE] // Load referrer 1460 mov x2, xSELF // pass Thread::Current 1461 bl \entrypoint // (uint32_t type_idx, Method* method, Thread*, SP) 1462 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1463 \return 1464END \name 1465.endm 1466 1467.macro TWO_ARG_REF_DOWNCALL name, entrypoint, return 1468 .extern \entrypoint 1469ENTRY \name 1470 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1471 ldr x2, [sp, #FRAME_SIZE_REFS_ONLY_CALLEE_SAVE] // Load referrer 1472 mov x3, xSELF // pass Thread::Current 1473 bl \entrypoint 1474 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1475 \return 1476END \name 1477.endm 1478 1479.macro THREE_ARG_REF_DOWNCALL name, entrypoint, return 1480 .extern \entrypoint 1481ENTRY \name 1482 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1483 ldr x3, [sp, #FRAME_SIZE_REFS_ONLY_CALLEE_SAVE] // Load referrer 1484 mov x4, xSELF // pass Thread::Current 1485 bl \entrypoint 1486 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1487 \return 1488END \name 1489.endm 1490 1491.macro RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1492 cbz w0, 1f // result zero branch over 1493 ret // return 14941: 1495 DELIVER_PENDING_EXCEPTION 1496.endm 1497 1498 /* 1499 * Entry from managed code that calls artHandleFillArrayDataFromCode and delivers exception on 1500 * failure. 1501 */ 1502TWO_ARG_REF_DOWNCALL art_quick_handle_fill_data, artHandleFillArrayDataFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1503 1504 /* 1505 * Entry from managed code when uninitialized static storage, this stub will run the class 1506 * initializer and deliver the exception on error. On success the static storage base is 1507 * returned. 1508 */ 1509ONE_ARG_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1510 1511ONE_ARG_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1512ONE_ARG_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1513 1514ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1515ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1516ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1517ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1518ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1519ONE_ARG_REF_DOWNCALL art_quick_get64_static, artGet64StaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1520ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1521 1522TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1523TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1524TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1525TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1526TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1527TWO_ARG_REF_DOWNCALL art_quick_get64_instance, artGet64InstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1528TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1529 1530TWO_ARG_REF_DOWNCALL art_quick_set8_static, artSet8StaticFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1531TWO_ARG_REF_DOWNCALL art_quick_set16_static, artSet16StaticFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1532TWO_ARG_REF_DOWNCALL art_quick_set32_static, artSet32StaticFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1533TWO_ARG_REF_DOWNCALL art_quick_set_obj_static, artSetObjStaticFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1534 1535THREE_ARG_REF_DOWNCALL art_quick_set8_instance, artSet8InstanceFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1536THREE_ARG_REF_DOWNCALL art_quick_set16_instance, artSet16InstanceFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1537THREE_ARG_REF_DOWNCALL art_quick_set32_instance, artSet32InstanceFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1538THREE_ARG_REF_DOWNCALL art_quick_set64_instance, artSet64InstanceFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1539THREE_ARG_REF_DOWNCALL art_quick_set_obj_instance, artSetObjInstanceFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1540 1541// This is separated out as the argument order is different. 1542 .extern artSet64StaticFromCode 1543ENTRY art_quick_set64_static 1544 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1545 ldr x1, [sp, #FRAME_SIZE_REFS_ONLY_CALLEE_SAVE] // Load referrer 1546 // x2 contains the parameter 1547 mov x3, xSELF // pass Thread::Current 1548 bl artSet64StaticFromCode 1549 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1550 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1551END art_quick_set64_static 1552 1553 /* 1554 * Entry from managed code to resolve a string, this stub will allocate a String and deliver an 1555 * exception on error. On success the String is returned. w0 holds the string index. The fast 1556 * path check for hit in strings cache has already been performed. 1557 */ 1558ONE_ARG_DOWNCALL art_quick_resolve_string, artResolveStringFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1559 1560// Generate the allocation entrypoints for each allocator. 1561GENERATE_ALLOC_ENTRYPOINTS_FOR_EACH_ALLOCATOR 1562 1563// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT(_rosalloc, RosAlloc). 1564ENTRY art_quick_alloc_object_rosalloc 1565 // Fast path rosalloc allocation. 1566 // x0: type_idx/return value, x1: ArtMethod*, xSELF(x19): Thread::Current 1567 // x2-x7: free. 1568 ldr x2, [x1, #ART_METHOD_DEX_CACHE_TYPES_OFFSET_64] // Load dex cache resolved types array 1569 // Load the class (x2) 1570 ldr w2, [x2, x0, lsl #COMPRESSED_REFERENCE_SIZE_SHIFT] 1571 cbz x2, .Lart_quick_alloc_object_rosalloc_slow_path // Check null class 1572 // Check class status. 1573 ldr w3, [x2, #MIRROR_CLASS_STATUS_OFFSET] 1574 cmp x3, #MIRROR_CLASS_STATUS_INITIALIZED 1575 bne .Lart_quick_alloc_object_rosalloc_slow_path 1576 // Add a fake dependence from the 1577 // following access flag and size 1578 // loads to the status load. 1579 // This is to prevent those loads 1580 // from being reordered above the 1581 // status load and reading wrong 1582 // values (an alternative is to use 1583 // a load-acquire for the status). 1584 eor x3, x3, x3 1585 add x2, x2, x3 1586 // Check access flags has 1587 // kAccClassIsFinalizable 1588 ldr w3, [x2, #MIRROR_CLASS_ACCESS_FLAGS_OFFSET] 1589 tst x3, #ACCESS_FLAGS_CLASS_IS_FINALIZABLE 1590 bne .Lart_quick_alloc_object_rosalloc_slow_path 1591 ldr x3, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] // Check if the thread local 1592 // allocation stack has room. 1593 // ldp won't work due to large offset. 1594 ldr x4, [xSELF, #THREAD_LOCAL_ALLOC_STACK_END_OFFSET] 1595 cmp x3, x4 1596 bhs .Lart_quick_alloc_object_rosalloc_slow_path 1597 ldr w3, [x2, #MIRROR_CLASS_OBJECT_SIZE_OFFSET] // Load the object size (x3) 1598 cmp x3, #ROSALLOC_MAX_THREAD_LOCAL_BRACKET_SIZE // Check if the size is for a thread 1599 // local allocation 1600 bhs .Lart_quick_alloc_object_rosalloc_slow_path 1601 // Compute the rosalloc bracket index 1602 // from the size. 1603 // Align up the size by the rosalloc 1604 // bracket quantum size and divide 1605 // by the quantum size and subtract 1606 // by 1. This code is a shorter but 1607 // equivalent version. 1608 sub x3, x3, #1 1609 lsr x3, x3, #ROSALLOC_BRACKET_QUANTUM_SIZE_SHIFT 1610 // Load the rosalloc run (x4) 1611 add x4, xSELF, x3, lsl #POINTER_SIZE_SHIFT 1612 ldr x4, [x4, #THREAD_ROSALLOC_RUNS_OFFSET] 1613 // Load the free list head (x3). This 1614 // will be the return val. 1615 ldr x3, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)] 1616 cbz x3, .Lart_quick_alloc_object_rosalloc_slow_path 1617 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber x0 and x1. 1618 ldr x1, [x3, #ROSALLOC_SLOT_NEXT_OFFSET] // Load the next pointer of the head 1619 // and update the list head with the 1620 // next pointer. 1621 str x1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)] 1622 // Store the class pointer in the 1623 // header. This also overwrites the 1624 // next pointer. The offsets are 1625 // asserted to match. 1626#if ROSALLOC_SLOT_NEXT_OFFSET != MIRROR_OBJECT_CLASS_OFFSET 1627#error "Class pointer needs to overwrite next pointer." 1628#endif 1629 POISON_HEAP_REF w2 1630 str w2, [x3, #MIRROR_OBJECT_CLASS_OFFSET] 1631 // Push the new object onto the thread 1632 // local allocation stack and 1633 // increment the thread local 1634 // allocation stack top. 1635 ldr x1, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] 1636 str w3, [x1], #COMPRESSED_REFERENCE_SIZE // (Increment x1 as a side effect.) 1637 str x1, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] 1638 // Decrement the size of the free list 1639 ldr w1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)] 1640 sub x1, x1, #1 1641 // TODO: consider combining this store 1642 // and the list head store above using 1643 // strd. 1644 str w1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)] 1645 // Fence. This is "ish" not "ishst" so 1646 // that the code after this allocation 1647 // site will see the right values in 1648 // the fields of the class. 1649 // Alternatively we could use "ishst" 1650 // if we use load-acquire for the 1651 // class status load.) 1652 dmb ish 1653 mov x0, x3 // Set the return value and return. 1654 ret 1655.Lart_quick_alloc_object_rosalloc_slow_path: 1656 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1657 mov x2, xSELF // pass Thread::Current 1658 bl artAllocObjectFromCodeRosAlloc // (uint32_t type_idx, Method* method, Thread*) 1659 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1660 RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1661END art_quick_alloc_object_rosalloc 1662 1663// The common fast path code for art_quick_alloc_object_tlab and art_quick_alloc_object_region_tlab. 1664// 1665// x0: type_idx/return value, x1: ArtMethod*, x2: Class*, xSELF(x19): Thread::Current 1666// x3-x7: free. 1667// Need to preserve x0 and x1 to the slow path. 1668.macro ALLOC_OBJECT_TLAB_FAST_PATH slowPathLabel 1669 cbz x2, \slowPathLabel // Check null class 1670 // Check class status. 1671 ldr w3, [x2, #MIRROR_CLASS_STATUS_OFFSET] 1672 cmp x3, #MIRROR_CLASS_STATUS_INITIALIZED 1673 bne \slowPathLabel 1674 // Add a fake dependence from the 1675 // following access flag and size 1676 // loads to the status load. 1677 // This is to prevent those loads 1678 // from being reordered above the 1679 // status load and reading wrong 1680 // values (an alternative is to use 1681 // a load-acquire for the status). 1682 eor x3, x3, x3 1683 add x2, x2, x3 1684 // Check access flags has 1685 // kAccClassIsFinalizable. 1686 ldr w3, [x2, #MIRROR_CLASS_ACCESS_FLAGS_OFFSET] 1687 tbnz x3, #ACCESS_FLAGS_CLASS_IS_FINALIZABLE_BIT, \slowPathLabel 1688 // Load thread_local_pos (x4) and 1689 // thread_local_end (x5). 1690 ldr x4, [xSELF, #THREAD_LOCAL_POS_OFFSET] 1691 ldr x5, [xSELF, #THREAD_LOCAL_END_OFFSET] 1692 sub x6, x5, x4 // Compute the remaining buf size. 1693 ldr w7, [x2, #MIRROR_CLASS_OBJECT_SIZE_OFFSET] // Load the object size (x7). 1694 cmp x7, x6 // Check if it fits. OK to do this 1695 // before rounding up the object size 1696 // assuming the buf size alignment. 1697 bhi \slowPathLabel 1698 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber x0 and x1. 1699 // Round up the object size by the 1700 // object alignment. (addr + 7) & ~7. 1701 add x7, x7, #OBJECT_ALIGNMENT_MASK 1702 and x7, x7, #OBJECT_ALIGNMENT_MASK_TOGGLED 1703 // Move old thread_local_pos to x0 1704 // for the return value. 1705 mov x0, x4 1706 add x5, x0, x7 1707 str x5, [xSELF, #THREAD_LOCAL_POS_OFFSET] // Store new thread_local_pos. 1708 ldr x5, [xSELF, #THREAD_LOCAL_OBJECTS_OFFSET] // Increment thread_local_objects. 1709 add x5, x5, #1 1710 str x5, [xSELF, #THREAD_LOCAL_OBJECTS_OFFSET] 1711 POISON_HEAP_REF w2 1712 str w2, [x0, #MIRROR_OBJECT_CLASS_OFFSET] // Store the class pointer. 1713 // Fence. This is "ish" not "ishst" so 1714 // that the code after this allocation 1715 // site will see the right values in 1716 // the fields of the class. 1717 // Alternatively we could use "ishst" 1718 // if we use load-acquire for the 1719 // class status load.) 1720 dmb ish 1721 ret 1722.endm 1723 1724// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT(_tlab, TLAB). 1725ENTRY art_quick_alloc_object_tlab 1726 // Fast path tlab allocation. 1727 // x0: type_idx/return value, x1: ArtMethod*, xSELF(x19): Thread::Current 1728 // x2-x7: free. 1729#if defined(USE_READ_BARRIER) 1730 mvn x0, xzr // Read barrier not supported here. 1731 ret // Return -1. 1732#endif 1733 ldr x2, [x1, #ART_METHOD_DEX_CACHE_TYPES_OFFSET_64] // Load dex cache resolved types array 1734 // Load the class (x2) 1735 ldr w2, [x2, x0, lsl #COMPRESSED_REFERENCE_SIZE_SHIFT] 1736 ALLOC_OBJECT_TLAB_FAST_PATH .Lart_quick_alloc_object_tlab_slow_path 1737.Lart_quick_alloc_object_tlab_slow_path: 1738 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // Save callee saves in case of GC. 1739 mov x2, xSELF // Pass Thread::Current. 1740 bl artAllocObjectFromCodeTLAB // (uint32_t type_idx, Method* method, Thread*) 1741 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1742 RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1743END art_quick_alloc_object_tlab 1744 1745// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT(_region_tlab, RegionTLAB) 1746ENTRY art_quick_alloc_object_region_tlab 1747 // Fast path region tlab allocation. 1748 // x0: type_idx/return value, x1: ArtMethod*, xSELF(x19): Thread::Current 1749 // x2-x7: free. 1750#if !defined(USE_READ_BARRIER) 1751 mvn x0, xzr // Read barrier must be enabled here. 1752 ret // Return -1. 1753#endif 1754 ldr x2, [x1, #ART_METHOD_DEX_CACHE_TYPES_OFFSET_64] // Load dex cache resolved types array 1755 // Load the class (x2) 1756 ldr w2, [x2, x0, lsl #COMPRESSED_REFERENCE_SIZE_SHIFT] 1757 // Read barrier for class load. 1758 ldr w3, [xSELF, #THREAD_IS_GC_MARKING_OFFSET] 1759 cbnz x3, .Lart_quick_alloc_object_region_tlab_class_load_read_barrier_slow_path 1760.Lart_quick_alloc_object_region_tlab_class_load_read_barrier_slow_path_exit: 1761 ALLOC_OBJECT_TLAB_FAST_PATH .Lart_quick_alloc_object_region_tlab_slow_path 1762.Lart_quick_alloc_object_region_tlab_class_load_read_barrier_slow_path: 1763 // The read barrier slow path. Mark 1764 // the class. 1765 stp x0, x1, [sp, #-32]! // Save registers (x0, x1, lr). 1766 str xLR, [sp, #16] // Align sp by 16 bytes. 1767 mov x0, x2 // Pass the class as the first param. 1768 bl artReadBarrierMark 1769 mov x2, x0 // Get the (marked) class back. 1770 ldp x0, x1, [sp, #0] // Restore registers. 1771 ldr xLR, [sp, #16] 1772 add sp, sp, #32 1773 b .Lart_quick_alloc_object_region_tlab_class_load_read_barrier_slow_path_exit 1774.Lart_quick_alloc_object_region_tlab_slow_path: 1775 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // Save callee saves in case of GC. 1776 mov x2, xSELF // Pass Thread::Current. 1777 bl artAllocObjectFromCodeRegionTLAB // (uint32_t type_idx, Method* method, Thread*) 1778 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME 1779 RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1780END art_quick_alloc_object_region_tlab 1781 1782 /* 1783 * Called by managed code when the thread has been asked to suspend. 1784 */ 1785 .extern artTestSuspendFromCode 1786ENTRY art_quick_test_suspend 1787 ldrh w0, [xSELF, #THREAD_FLAGS_OFFSET] // get xSELF->state_and_flags.as_struct.flags 1788 cbnz w0, .Lneed_suspend // check flags == 0 1789 ret // return if flags == 0 1790.Lneed_suspend: 1791 mov x0, xSELF 1792 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves for stack crawl 1793 bl artTestSuspendFromCode // (Thread*) 1794 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME_AND_RETURN 1795END art_quick_test_suspend 1796 1797ENTRY art_quick_implicit_suspend 1798 mov x0, xSELF 1799 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME // save callee saves for stack crawl 1800 bl artTestSuspendFromCode // (Thread*) 1801 RESTORE_REFS_ONLY_CALLEE_SAVE_FRAME_AND_RETURN 1802END art_quick_implicit_suspend 1803 1804 /* 1805 * Called by managed code that is attempting to call a method on a proxy class. On entry 1806 * x0 holds the proxy method and x1 holds the receiver; The frame size of the invoked proxy 1807 * method agrees with a ref and args callee save frame. 1808 */ 1809 .extern artQuickProxyInvokeHandler 1810ENTRY art_quick_proxy_invoke_handler 1811 SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME_WITH_METHOD_IN_X0 1812 mov x2, xSELF // pass Thread::Current 1813 mov x3, sp // pass SP 1814 bl artQuickProxyInvokeHandler // (Method* proxy method, receiver, Thread*, SP) 1815 ldr x2, [xSELF, THREAD_EXCEPTION_OFFSET] 1816 cbnz x2, .Lexception_in_proxy // success if no exception is pending 1817 RESTORE_REFS_AND_ARGS_CALLEE_SAVE_FRAME // Restore frame 1818 fmov d0, x0 // Store result in d0 in case it was float or double 1819 ret // return on success 1820.Lexception_in_proxy: 1821 RESTORE_REFS_AND_ARGS_CALLEE_SAVE_FRAME 1822 DELIVER_PENDING_EXCEPTION 1823END art_quick_proxy_invoke_handler 1824 1825 /* 1826 * Called to resolve an imt conflict. 1827 * x0 is the conflict ArtMethod. 1828 * xIP1 is a hidden argument that holds the target interface method's dex method index. 1829 * 1830 * Note that this stub writes to xIP0, xIP1, and x0. 1831 */ 1832 .extern artInvokeInterfaceTrampoline 1833ENTRY art_quick_imt_conflict_trampoline 1834 ldr xIP0, [sp, #0] // Load referrer 1835 ldr xIP0, [xIP0, #ART_METHOD_DEX_CACHE_METHODS_OFFSET_64] // Load dex cache methods array 1836 ldr xIP0, [xIP0, xIP1, lsl #POINTER_SIZE_SHIFT] // Load interface method 1837 ldr xIP1, [x0, #ART_METHOD_JNI_OFFSET_64] // Load ImtConflictTable 1838 ldr x0, [xIP1] // Load first entry in ImtConflictTable. 1839.Limt_table_iterate: 1840 cmp x0, xIP0 1841 // Branch if found. Benchmarks have shown doing a branch here is better. 1842 beq .Limt_table_found 1843 // If the entry is null, the interface method is not in the ImtConflictTable. 1844 cbz x0, .Lconflict_trampoline 1845 // Iterate over the entries of the ImtConflictTable. 1846 ldr x0, [xIP1, #(2 * __SIZEOF_POINTER__)]! 1847 b .Limt_table_iterate 1848.Limt_table_found: 1849 // We successfully hit an entry in the table. Load the target method 1850 // and jump to it. 1851 ldr x0, [xIP1, #__SIZEOF_POINTER__] 1852 ldr xIP0, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 1853 br xIP0 1854.Lconflict_trampoline: 1855 // Call the runtime stub to populate the ImtConflictTable and jump to the 1856 // resolved method. 1857 INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline 1858END art_quick_imt_conflict_trampoline 1859 1860ENTRY art_quick_resolution_trampoline 1861 SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME 1862 mov x2, xSELF 1863 mov x3, sp 1864 bl artQuickResolutionTrampoline // (called, receiver, Thread*, SP) 1865 cbz x0, 1f 1866 mov xIP0, x0 // Remember returned code pointer in xIP0. 1867 ldr x0, [sp, #0] // artQuickResolutionTrampoline puts called method in *SP. 1868 RESTORE_REFS_AND_ARGS_CALLEE_SAVE_FRAME 1869 br xIP0 18701: 1871 RESTORE_REFS_AND_ARGS_CALLEE_SAVE_FRAME 1872 DELIVER_PENDING_EXCEPTION 1873END art_quick_resolution_trampoline 1874 1875/* 1876 * Generic JNI frame layout: 1877 * 1878 * #-------------------# 1879 * | | 1880 * | caller method... | 1881 * #-------------------# <--- SP on entry 1882 * | Return X30/LR | 1883 * | X29/FP | callee save 1884 * | X28 | callee save 1885 * | X27 | callee save 1886 * | X26 | callee save 1887 * | X25 | callee save 1888 * | X24 | callee save 1889 * | X23 | callee save 1890 * | X22 | callee save 1891 * | X21 | callee save 1892 * | X20 | callee save 1893 * | X19 | callee save 1894 * | X7 | arg7 1895 * | X6 | arg6 1896 * | X5 | arg5 1897 * | X4 | arg4 1898 * | X3 | arg3 1899 * | X2 | arg2 1900 * | X1 | arg1 1901 * | D7 | float arg 8 1902 * | D6 | float arg 7 1903 * | D5 | float arg 6 1904 * | D4 | float arg 5 1905 * | D3 | float arg 4 1906 * | D2 | float arg 3 1907 * | D1 | float arg 2 1908 * | D0 | float arg 1 1909 * | Method* | <- X0 1910 * #-------------------# 1911 * | local ref cookie | // 4B 1912 * | handle scope size | // 4B 1913 * #-------------------# 1914 * | JNI Call Stack | 1915 * #-------------------# <--- SP on native call 1916 * | | 1917 * | Stack for Regs | The trampoline assembly will pop these values 1918 * | | into registers for native call 1919 * #-------------------# 1920 * | Native code ptr | 1921 * #-------------------# 1922 * | Free scratch | 1923 * #-------------------# 1924 * | Ptr to (1) | <--- SP 1925 * #-------------------# 1926 */ 1927 /* 1928 * Called to do a generic JNI down-call 1929 */ 1930ENTRY art_quick_generic_jni_trampoline 1931 SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME_WITH_METHOD_IN_X0 1932 1933 // Save SP , so we can have static CFI info. 1934 mov x28, sp 1935 .cfi_def_cfa_register x28 1936 1937 // This looks the same, but is different: this will be updated to point to the bottom 1938 // of the frame when the handle scope is inserted. 1939 mov xFP, sp 1940 1941 mov xIP0, #5120 1942 sub sp, sp, xIP0 1943 1944 // prepare for artQuickGenericJniTrampoline call 1945 // (Thread*, SP) 1946 // x0 x1 <= C calling convention 1947 // xSELF xFP <= where they are 1948 1949 mov x0, xSELF // Thread* 1950 mov x1, xFP 1951 bl artQuickGenericJniTrampoline // (Thread*, sp) 1952 1953 // The C call will have registered the complete save-frame on success. 1954 // The result of the call is: 1955 // x0: pointer to native code, 0 on error. 1956 // x1: pointer to the bottom of the used area of the alloca, can restore stack till there. 1957 1958 // Check for error = 0. 1959 cbz x0, .Lexception_in_native 1960 1961 // Release part of the alloca. 1962 mov sp, x1 1963 1964 // Save the code pointer 1965 mov xIP0, x0 1966 1967 // Load parameters from frame into registers. 1968 // TODO Check with artQuickGenericJniTrampoline. 1969 // Also, check again APPCS64 - the stack arguments are interleaved. 1970 ldp x0, x1, [sp] 1971 ldp x2, x3, [sp, #16] 1972 ldp x4, x5, [sp, #32] 1973 ldp x6, x7, [sp, #48] 1974 1975 ldp d0, d1, [sp, #64] 1976 ldp d2, d3, [sp, #80] 1977 ldp d4, d5, [sp, #96] 1978 ldp d6, d7, [sp, #112] 1979 1980 add sp, sp, #128 1981 1982 blr xIP0 // native call. 1983 1984 // result sign extension is handled in C code 1985 // prepare for artQuickGenericJniEndTrampoline call 1986 // (Thread*, result, result_f) 1987 // x0 x1 x2 <= C calling convention 1988 mov x1, x0 // Result (from saved). 1989 mov x0, xSELF // Thread register. 1990 fmov x2, d0 // d0 will contain floating point result, but needs to go into x2 1991 1992 bl artQuickGenericJniEndTrampoline 1993 1994 // Pending exceptions possible. 1995 ldr x2, [xSELF, THREAD_EXCEPTION_OFFSET] 1996 cbnz x2, .Lexception_in_native 1997 1998 // Tear down the alloca. 1999 mov sp, x28 2000 .cfi_def_cfa_register sp 2001 2002 // Tear down the callee-save frame. 2003 RESTORE_REFS_AND_ARGS_CALLEE_SAVE_FRAME 2004 2005 // store into fpr, for when it's a fpr return... 2006 fmov d0, x0 2007 ret 2008 2009.Lexception_in_native: 2010 // Move to x1 then sp to please assembler. 2011 ldr x1, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 2012 mov sp, x1 2013 .cfi_def_cfa_register sp 2014 # This will create a new save-all frame, required by the runtime. 2015 DELIVER_PENDING_EXCEPTION 2016END art_quick_generic_jni_trampoline 2017 2018/* 2019 * Called to bridge from the quick to interpreter ABI. On entry the arguments match those 2020 * of a quick call: 2021 * x0 = method being called/to bridge to. 2022 * x1..x7, d0..d7 = arguments to that method. 2023 */ 2024ENTRY art_quick_to_interpreter_bridge 2025 SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME // Set up frame and save arguments. 2026 2027 // x0 will contain mirror::ArtMethod* method. 2028 mov x1, xSELF // How to get Thread::Current() ??? 2029 mov x2, sp 2030 2031 // uint64_t artQuickToInterpreterBridge(mirror::ArtMethod* method, Thread* self, 2032 // mirror::ArtMethod** sp) 2033 bl artQuickToInterpreterBridge 2034 2035 RESTORE_REFS_AND_ARGS_CALLEE_SAVE_FRAME // TODO: no need to restore arguments in this case. 2036 2037 fmov d0, x0 2038 2039 RETURN_OR_DELIVER_PENDING_EXCEPTION 2040END art_quick_to_interpreter_bridge 2041 2042 2043// 2044// Instrumentation-related stubs 2045// 2046 .extern artInstrumentationMethodEntryFromCode 2047ENTRY art_quick_instrumentation_entry 2048 SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME 2049 2050 mov x20, x0 // Preserve method reference in a callee-save. 2051 2052 mov x2, xSELF 2053 mov x3, xLR 2054 bl artInstrumentationMethodEntryFromCode // (Method*, Object*, Thread*, LR) 2055 2056 mov xIP0, x0 // x0 = result of call. 2057 mov x0, x20 // Reload method reference. 2058 2059 RESTORE_REFS_AND_ARGS_CALLEE_SAVE_FRAME // Note: will restore xSELF 2060 adr xLR, art_quick_instrumentation_exit 2061 br xIP0 // Tail-call method with lr set to art_quick_instrumentation_exit. 2062END art_quick_instrumentation_entry 2063 2064 .extern artInstrumentationMethodExitFromCode 2065ENTRY art_quick_instrumentation_exit 2066 mov xLR, #0 // Clobber LR for later checks. 2067 2068 SETUP_REFS_ONLY_CALLEE_SAVE_FRAME 2069 2070 // We need to save x0 and d0. We could use a callee-save from SETUP_REF_ONLY, but then 2071 // we would need to fully restore it. As there are a lot of callee-save registers, it seems 2072 // easier to have an extra small stack area. 2073 2074 str x0, [sp, #-16]! // Save integer result. 2075 .cfi_adjust_cfa_offset 16 2076 str d0, [sp, #8] // Save floating-point result. 2077 2078 add x1, sp, #16 // Pass SP. 2079 mov x2, x0 // Pass integer result. 2080 fmov x3, d0 // Pass floating-point result. 2081 mov x0, xSELF // Pass Thread. 2082 bl artInstrumentationMethodExitFromCode // (Thread*, SP, gpr_res, fpr_res) 2083 2084 mov xIP0, x0 // Return address from instrumentation call. 2085 mov xLR, x1 // r1 is holding link register if we're to bounce to deoptimize 2086 2087 ldr d0, [sp, #8] // Restore floating-point result. 2088 ldr x0, [sp], 16 // Restore integer result, and drop stack area. 2089 .cfi_adjust_cfa_offset 16 2090 2091 POP_REFS_ONLY_CALLEE_SAVE_FRAME 2092 2093 br xIP0 // Tail-call out. 2094END art_quick_instrumentation_exit 2095 2096 /* 2097 * Instrumentation has requested that we deoptimize into the interpreter. The deoptimization 2098 * will long jump to the upcall with a special exception of -1. 2099 */ 2100 .extern artDeoptimize 2101ENTRY art_quick_deoptimize 2102 SETUP_SAVE_ALL_CALLEE_SAVE_FRAME 2103 mov x0, xSELF // Pass thread. 2104 bl artDeoptimize // artDeoptimize(Thread*) 2105 brk 0 2106END art_quick_deoptimize 2107 2108 /* 2109 * Compiled code has requested that we deoptimize into the interpreter. The deoptimization 2110 * will long jump to the upcall with a special exception of -1. 2111 */ 2112 .extern artDeoptimizeFromCompiledCode 2113ENTRY art_quick_deoptimize_from_compiled_code 2114 SETUP_SAVE_ALL_CALLEE_SAVE_FRAME 2115 mov x0, xSELF // Pass thread. 2116 bl artDeoptimizeFromCompiledCode // artDeoptimizeFromCompiledCode(Thread*) 2117 brk 0 2118END art_quick_deoptimize_from_compiled_code 2119 2120 2121 /* 2122 * String's indexOf. 2123 * 2124 * TODO: Not very optimized. 2125 * On entry: 2126 * x0: string object (known non-null) 2127 * w1: char to match (known <= 0xFFFF) 2128 * w2: Starting offset in string data 2129 */ 2130ENTRY art_quick_indexof 2131 ldr w3, [x0, #MIRROR_STRING_COUNT_OFFSET] 2132 add x0, x0, #MIRROR_STRING_VALUE_OFFSET 2133 2134 /* Clamp start to [0..count] */ 2135 cmp w2, #0 2136 csel w2, wzr, w2, lt 2137 cmp w2, w3 2138 csel w2, w3, w2, gt 2139 2140 /* Save a copy to compute result */ 2141 mov x5, x0 2142 2143 /* Build pointer to start of data to compare and pre-bias */ 2144 add x0, x0, x2, lsl #1 2145 sub x0, x0, #2 2146 2147 /* Compute iteration count */ 2148 sub w2, w3, w2 2149 2150 /* 2151 * At this point we have: 2152 * x0: start of the data to test 2153 * w1: char to compare 2154 * w2: iteration count 2155 * x5: original start of string data 2156 */ 2157 2158 subs w2, w2, #4 2159 b.lt .Lindexof_remainder 2160 2161.Lindexof_loop4: 2162 ldrh w6, [x0, #2]! 2163 ldrh w7, [x0, #2]! 2164 ldrh wIP0, [x0, #2]! 2165 ldrh wIP1, [x0, #2]! 2166 cmp w6, w1 2167 b.eq .Lmatch_0 2168 cmp w7, w1 2169 b.eq .Lmatch_1 2170 cmp wIP0, w1 2171 b.eq .Lmatch_2 2172 cmp wIP1, w1 2173 b.eq .Lmatch_3 2174 subs w2, w2, #4 2175 b.ge .Lindexof_loop4 2176 2177.Lindexof_remainder: 2178 adds w2, w2, #4 2179 b.eq .Lindexof_nomatch 2180 2181.Lindexof_loop1: 2182 ldrh w6, [x0, #2]! 2183 cmp w6, w1 2184 b.eq .Lmatch_3 2185 subs w2, w2, #1 2186 b.ne .Lindexof_loop1 2187 2188.Lindexof_nomatch: 2189 mov x0, #-1 2190 ret 2191 2192.Lmatch_0: 2193 sub x0, x0, #6 2194 sub x0, x0, x5 2195 asr x0, x0, #1 2196 ret 2197.Lmatch_1: 2198 sub x0, x0, #4 2199 sub x0, x0, x5 2200 asr x0, x0, #1 2201 ret 2202.Lmatch_2: 2203 sub x0, x0, #2 2204 sub x0, x0, x5 2205 asr x0, x0, #1 2206 ret 2207.Lmatch_3: 2208 sub x0, x0, x5 2209 asr x0, x0, #1 2210 ret 2211END art_quick_indexof 2212