quick_entrypoints_arm64.S revision 41c507a9dae44b8329a857da3d9810fab2e9ddc6
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "asm_support_arm64.S" 18 19#include "arch/quick_alloc_entrypoints.S" 20 21 22 /* 23 * Macro that sets up the callee save frame to conform with 24 * Runtime::CreateCalleeSaveMethod(kSaveAll) 25 */ 26.macro SETUP_SAVE_ALL_CALLEE_SAVE_FRAME 27 adrp x9, :got:_ZN3art7Runtime9instance_E 28 ldr x9, [x9, #:got_lo12:_ZN3art7Runtime9instance_E] 29 30 // Our registers aren't intermixed - just spill in order. 31 ldr x9,[x9] // x9 = & (art::Runtime * art::Runtime.instance_) . 32 33 // x9 = (ArtMethod*) Runtime.instance_.callee_save_methods[kRefAndArgs] . 34 ldr x9, [x9, RUNTIME_SAVE_ALL_CALLEE_SAVE_FRAME_OFFSET ] 35 36 sub sp, sp, #368 37 .cfi_adjust_cfa_offset 368 38 39 // Ugly compile-time check, but we only have the preprocessor. 40#if (FRAME_SIZE_SAVE_ALL_CALLEE_SAVE != 368) 41#error "SAVE_ALL_CALLEE_SAVE_FRAME(ARM64) size not as expected." 42#endif 43 44 // FP args 45 stp d0, d1, [sp, #8] 46 stp d2, d3, [sp, #24] 47 stp d4, d5, [sp, #40] 48 stp d6, d7, [sp, #56] 49 50 // FP callee-saves 51 stp d8, d9, [sp, #72] 52 stp d10, d11, [sp, #88] 53 stp d12, d13, [sp, #104] 54 stp d14, d15, [sp, #120] 55 56 stp d16, d17, [sp, #136] 57 stp d18, d19, [sp, #152] 58 stp d20, d21, [sp, #168] 59 stp d22, d23, [sp, #184] 60 stp d24, d25, [sp, #200] 61 stp d26, d27, [sp, #216] 62 stp d28, d29, [sp, #232] 63 stp d30, d31, [sp, #248] 64 65 66 // Callee saved. 67 stp xSELF, x19, [sp, #264] 68 .cfi_rel_offset x18, 264 69 .cfi_rel_offset x19, 272 70 71 stp x20, x21, [sp, #280] 72 .cfi_rel_offset x20, 280 73 .cfi_rel_offset x21, 288 74 75 stp x22, x23, [sp, #296] 76 .cfi_rel_offset x22, 296 77 .cfi_rel_offset x23, 304 78 79 stp x24, x25, [sp, #312] 80 .cfi_rel_offset x24, 312 81 .cfi_rel_offset x25, 320 82 83 stp x26, x27, [sp, #328] 84 .cfi_rel_offset x26, 328 85 .cfi_rel_offset x27, 336 86 87 stp x28, xFP, [sp, #344] // Save FP. 88 .cfi_rel_offset x28, 344 89 .cfi_rel_offset x29, 352 90 91 str xLR, [sp, #360] 92 .cfi_rel_offset x30, 360 93 94 // Loads appropriate callee-save-method 95 str x9, [sp] // Store ArtMethod* Runtime::callee_save_methods_[kRefsAndArgs] 96 97.endm 98 99 /* 100 * Macro that sets up the callee save frame to conform with 101 * Runtime::CreateCalleeSaveMethod(kRefsOnly). 102 */ 103// WIP. 104.macro SETUP_REF_ONLY_CALLEE_SAVE_FRAME 105 adrp x9, :got:_ZN3art7Runtime9instance_E 106 ldr x9, [x9, #:got_lo12:_ZN3art7Runtime9instance_E] 107 108 // Our registers aren't intermixed - just spill in order. 109 ldr x9,[x9] // x9 = & (art::Runtime * art::Runtime.instance_) . 110 111 // x9 = (ArtMethod*) Runtime.instance_.callee_save_methods[kRefAndArgs] . 112 ldr x9, [x9, RUNTIME_REFS_ONLY_CALLEE_SAVE_FRAME_OFFSET ] 113 114 sub sp, sp, #176 115 .cfi_adjust_cfa_offset 176 116 117 // Ugly compile-time check, but we only have the preprocessor. 118#if (FRAME_SIZE_REFS_ONLY_CALLEE_SAVE != 176) 119#error "REFS_ONLY_CALLEE_SAVE_FRAME(ARM64) size not as expected." 120#endif 121 122 // FP callee-saves 123 stp d8, d9, [sp, #8] 124 stp d10, d11, [sp, #24] 125 stp d12, d13, [sp, #40] 126 stp d14, d15, [sp, #56] 127 128 // Callee saved. 129 stp xSELF, x19, [sp, #72] 130 .cfi_rel_offset x18, 72 131 .cfi_rel_offset x19, 80 132 133 stp x20, x21, [sp, #88] 134 .cfi_rel_offset x20, 88 135 .cfi_rel_offset x21, 96 136 137 stp x22, x23, [sp, #104] 138 .cfi_rel_offset x22, 104 139 .cfi_rel_offset x23, 112 140 141 stp x24, x25, [sp, #120] 142 .cfi_rel_offset x24, 120 143 .cfi_rel_offset x25, 128 144 145 stp x26, x27, [sp, #136] 146 .cfi_rel_offset x26, 136 147 .cfi_rel_offset x27, 144 148 149 stp x28, xFP, [sp, #152] // Save FP. 150 .cfi_rel_offset x28, 152 151 .cfi_rel_offset x29, 160 152 153 str xLR, [sp, #168] 154 .cfi_rel_offset x30, 168 155 156 // Loads appropriate callee-save-method 157 str x9, [sp] // Store ArtMethod* Runtime::callee_save_methods_[kRefsAndArgs] 158.endm 159 160.macro RESTORE_REF_ONLY_CALLEE_SAVE_FRAME 161 // FP callee saves 162 ldp d8, d9, [sp, #8] 163 ldp d10, d11, [sp, #24] 164 ldp d12, d13, [sp, #40] 165 ldp d14, d15, [sp, #56] 166 167 // Callee saved. 168 ldp xSELF, x19, [sp, #72] 169 .cfi_restore x18 170 .cfi_restore x19 171 172 ldp x20, x21, [sp, #88] 173 .cfi_restore x20 174 .cfi_restore x21 175 176 ldp x22, x23, [sp, #104] 177 .cfi_restore x22 178 .cfi_restore x23 179 180 ldp x24, x25, [sp, #120] 181 .cfi_restore x24 182 .cfi_restore x25 183 184 ldp x26, x27, [sp, #136] 185 .cfi_restore x26 186 .cfi_restore x27 187 188 ldp x28, xFP, [sp, #152] // Save FP. 189 .cfi_restore x28 190 .cfi_restore x29 191 192 ldr xLR, [sp, #168] 193 .cfi_restore x30 194 195 add sp, sp, #176 196 .cfi_adjust_cfa_offset -176 197.endm 198 199.macro RESTORE_REF_ONLY_CALLEE_SAVE_FRAME_AND_RETURN 200 RESTORE_REF_ONLY_CALLEE_SAVE_FRAME 201 ret 202.endm 203 204 205.macro SETUP_REF_AND_ARGS_CALLEE_SAVE_FRAME_INTERNAL 206 sub sp, sp, #304 207 .cfi_adjust_cfa_offset 304 208 209 // Ugly compile-time check, but we only have the preprocessor. 210#if (FRAME_SIZE_REFS_AND_ARGS_CALLEE_SAVE != 304) 211#error "REFS_AND_ARGS_CALLEE_SAVE_FRAME(ARM64) size not as expected." 212#endif 213 214 stp d0, d1, [sp, #16] 215 stp d2, d3, [sp, #32] 216 stp d4, d5, [sp, #48] 217 stp d6, d7, [sp, #64] 218 stp d8, d9, [sp, #80] 219 stp d10, d11, [sp, #96] 220 stp d12, d13, [sp, #112] 221 stp d14, d15, [sp, #128] 222 223 stp x1, x2, [sp, #144] 224 .cfi_rel_offset x1, 144 225 .cfi_rel_offset x2, 152 226 227 stp x3, x4, [sp, #160] 228 .cfi_rel_offset x3, 160 229 .cfi_rel_offset x4, 168 230 231 stp x5, x6, [sp, #176] 232 .cfi_rel_offset x5, 176 233 .cfi_rel_offset x6, 184 234 235 stp x7, xSELF, [sp, #192] 236 .cfi_rel_offset x7, 192 237 .cfi_rel_offset x18, 200 238 239 stp x19, x20, [sp, #208] 240 .cfi_rel_offset x19, 208 241 .cfi_rel_offset x20, 216 242 243 stp x21, x22, [sp, #224] 244 .cfi_rel_offset x21, 224 245 .cfi_rel_offset x22, 232 246 247 stp x23, x24, [sp, #240] 248 .cfi_rel_offset x23, 240 249 .cfi_rel_offset x24, 248 250 251 stp x25, x26, [sp, #256] 252 .cfi_rel_offset x25, 256 253 .cfi_rel_offset x26, 264 254 255 stp x27, x28, [sp, #272] 256 .cfi_rel_offset x27, 272 257 .cfi_rel_offset x28, 280 258 259 stp xFP, xLR, [sp, #288] 260 .cfi_rel_offset x29, 288 261 .cfi_rel_offset x30, 296 262.endm 263 264 /* 265 * Macro that sets up the callee save frame to conform with 266 * Runtime::CreateCalleeSaveMethod(kRefsAndArgs). 267 * 268 * TODO This is probably too conservative - saving FP & LR. 269 */ 270.macro SETUP_REF_AND_ARGS_CALLEE_SAVE_FRAME 271 adrp x9, :got:_ZN3art7Runtime9instance_E 272 ldr x9, [x9, #:got_lo12:_ZN3art7Runtime9instance_E] 273 274 // Our registers aren't intermixed - just spill in order. 275 ldr x9,[x9] // x9 = & (art::Runtime * art::Runtime.instance_) . 276 277 // x9 = (ArtMethod*) Runtime.instance_.callee_save_methods[kRefAndArgs] . 278 ldr x9, [x9, RUNTIME_REF_AND_ARGS_CALLEE_SAVE_FRAME_OFFSET ] 279 280 SETUP_REF_AND_ARGS_CALLEE_SAVE_FRAME_INTERNAL 281 282 str x9, [sp] // Store ArtMethod* Runtime::callee_save_methods_[kRefsAndArgs] 283.endm 284 285.macro RESTORE_REF_AND_ARGS_CALLEE_SAVE_FRAME 286 287 ldp d0, d1, [sp, #16] 288 ldp d2, d3, [sp, #32] 289 ldp d4, d5, [sp, #48] 290 ldp d6, d7, [sp, #64] 291 ldp d8, d9, [sp, #80] 292 ldp d10, d11, [sp, #96] 293 ldp d12, d13, [sp, #112] 294 ldp d14, d15, [sp, #128] 295 296 // args. 297 ldp x1, x2, [sp, #144] 298 .cfi_restore x1 299 .cfi_restore x2 300 301 ldp x3, x4, [sp, #160] 302 .cfi_restore x3 303 .cfi_restore x4 304 305 ldp x5, x6, [sp, #176] 306 .cfi_restore x5 307 .cfi_restore x6 308 309 ldp x7, xSELF, [sp, #192] 310 .cfi_restore x7 311 .cfi_restore x18 312 313 ldp x19, x20, [sp, #208] 314 .cfi_restore x19 315 .cfi_restore x20 316 317 ldp x21, x22, [sp, #224] 318 .cfi_restore x21 319 .cfi_restore x22 320 321 ldp x23, x24, [sp, #240] 322 .cfi_restore x23 323 .cfi_restore x24 324 325 ldp x25, x26, [sp, #256] 326 .cfi_restore x25 327 .cfi_restore x26 328 329 ldp x27, x28, [sp, #272] 330 .cfi_restore x27 331 .cfi_restore x28 332 333 ldp xFP, xLR, [sp, #288] 334 .cfi_restore x29 335 .cfi_restore x30 336 337 add sp, sp, #304 338 .cfi_adjust_cfa_offset -304 339.endm 340 341.macro RETURN_IF_RESULT_IS_ZERO 342 cbnz x0, 1f // result non-zero branch over 343 ret // return 3441: 345.endm 346 347.macro RETURN_IF_RESULT_IS_NON_ZERO 348 cbz x0, 1f // result zero branch over 349 ret // return 3501: 351.endm 352 353 /* 354 * Macro that set calls through to artDeliverPendingExceptionFromCode, where the pending 355 * exception is Thread::Current()->exception_ 356 */ 357.macro DELIVER_PENDING_EXCEPTION 358 SETUP_SAVE_ALL_CALLEE_SAVE_FRAME 359 mov x0, xSELF 360 mov x1, sp 361 362 // Point of no return. 363 b artDeliverPendingExceptionFromCode // artDeliverPendingExceptionFromCode(Thread*, SP) 364 brk 0 // Unreached 365.endm 366 367.macro RETURN_OR_DELIVER_PENDING_EXCEPTION_REG reg 368 ldr \reg, [xSELF, # THREAD_EXCEPTION_OFFSET] // Get exception field. 369 cbnz \reg, 1f 370 ret 3711: 372 DELIVER_PENDING_EXCEPTION 373.endm 374 375.macro RETURN_OR_DELIVER_PENDING_EXCEPTION 376 RETURN_OR_DELIVER_PENDING_EXCEPTION_REG x9 377.endm 378 379// Same as above with x1. This is helpful in stubs that want to avoid clobbering another register. 380.macro RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 381 RETURN_OR_DELIVER_PENDING_EXCEPTION_REG x1 382.endm 383 384.macro RETURN_IF_W0_IS_ZERO_OR_DELIVER 385 cbnz w0, 1f // result non-zero branch over 386 ret // return 3871: 388 DELIVER_PENDING_EXCEPTION 389.endm 390 391.macro NO_ARG_RUNTIME_EXCEPTION c_name, cxx_name 392 .extern \cxx_name 393ENTRY \c_name 394 SETUP_SAVE_ALL_CALLEE_SAVE_FRAME // save all registers as basis for long jump context 395 mov x0, xSELF // pass Thread::Current 396 mov x1, sp // pass SP 397 b \cxx_name // \cxx_name(Thread*, SP) 398END \c_name 399.endm 400 401.macro ONE_ARG_RUNTIME_EXCEPTION c_name, cxx_name 402 .extern \cxx_name 403ENTRY \c_name 404 SETUP_SAVE_ALL_CALLEE_SAVE_FRAME // save all registers as basis for long jump context. 405 mov x1, xSELF // pass Thread::Current. 406 mov x2, sp // pass SP. 407 b \cxx_name // \cxx_name(arg, Thread*, SP). 408 brk 0 409END \c_name 410.endm 411 412.macro TWO_ARG_RUNTIME_EXCEPTION c_name, cxx_name 413 .extern \cxx_name 414ENTRY \c_name 415 SETUP_SAVE_ALL_CALLEE_SAVE_FRAME // save all registers as basis for long jump context 416 mov x2, xSELF // pass Thread::Current 417 mov x3, sp // pass SP 418 b \cxx_name // \cxx_name(arg1, arg2, Thread*, SP) 419 brk 0 420END \c_name 421.endm 422 423 /* 424 * Called by managed code, saves callee saves and then calls artThrowException 425 * that will place a mock Method* at the bottom of the stack. Arg1 holds the exception. 426 */ 427ONE_ARG_RUNTIME_EXCEPTION art_quick_deliver_exception, artDeliverExceptionFromCode 428 429 /* 430 * Called by managed code to create and deliver a NullPointerException. 431 */ 432NO_ARG_RUNTIME_EXCEPTION art_quick_throw_null_pointer_exception, artThrowNullPointerExceptionFromCode 433 434 /* 435 * Called by managed code to create and deliver an ArithmeticException. 436 */ 437NO_ARG_RUNTIME_EXCEPTION art_quick_throw_div_zero, artThrowDivZeroFromCode 438 439 /* 440 * Called by managed code to create and deliver an ArrayIndexOutOfBoundsException. Arg1 holds 441 * index, arg2 holds limit. 442 */ 443TWO_ARG_RUNTIME_EXCEPTION art_quick_throw_array_bounds, artThrowArrayBoundsFromCode 444 445 /* 446 * Called by managed code to create and deliver a StackOverflowError. 447 */ 448NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFromCode 449 450 /* 451 * Called by managed code to create and deliver a NoSuchMethodError. 452 */ 453ONE_ARG_RUNTIME_EXCEPTION art_quick_throw_no_such_method, artThrowNoSuchMethodFromCode 454 455 /* 456 * All generated callsites for interface invokes and invocation slow paths will load arguments 457 * as usual - except instead of loading arg0/x0 with the target Method*, arg0/x0 will contain 458 * the method_idx. This wrapper will save arg1-arg3, load the caller's Method*, align the 459 * stack and call the appropriate C helper. 460 * NOTE: "this" is first visible argument of the target, and so can be found in arg1/x1. 461 * 462 * The helper will attempt to locate the target and return a 128-bit result in x0/x1 consisting 463 * of the target Method* in x0 and method->code_ in x1. 464 * 465 * If unsuccessful, the helper will return NULL/????. There will be a pending exception in the 466 * thread and we branch to another stub to deliver it. 467 * 468 * On success this wrapper will restore arguments and *jump* to the target, leaving the lr 469 * pointing back to the original caller. 470 * 471 * Adapted from ARM32 code. 472 * 473 * Clobbers x12. 474 */ 475.macro INVOKE_TRAMPOLINE c_name, cxx_name 476 .extern \cxx_name 477ENTRY \c_name 478 SETUP_REF_AND_ARGS_CALLEE_SAVE_FRAME // save callee saves in case allocation triggers GC 479 // Helper signature is always 480 // (method_idx, *this_object, *caller_method, *self, sp) 481 482 ldr w2, [sp, #FRAME_SIZE_REFS_AND_ARGS_CALLEE_SAVE] // pass caller Method* 483 mov x3, xSELF // pass Thread::Current 484 mov x4, sp 485 bl \cxx_name // (method_idx, this, caller, Thread*, SP) 486 mov x12, x1 // save Method*->code_ 487 RESTORE_REF_AND_ARGS_CALLEE_SAVE_FRAME 488 cbz x0, 1f // did we find the target? if not go to exception delivery 489 br x12 // tail call to target 4901: 491 DELIVER_PENDING_EXCEPTION 492END \c_name 493.endm 494 495INVOKE_TRAMPOLINE art_quick_invoke_interface_trampoline, artInvokeInterfaceTrampoline 496INVOKE_TRAMPOLINE art_quick_invoke_interface_trampoline_with_access_check, artInvokeInterfaceTrampolineWithAccessCheck 497 498INVOKE_TRAMPOLINE art_quick_invoke_static_trampoline_with_access_check, artInvokeStaticTrampolineWithAccessCheck 499INVOKE_TRAMPOLINE art_quick_invoke_direct_trampoline_with_access_check, artInvokeDirectTrampolineWithAccessCheck 500INVOKE_TRAMPOLINE art_quick_invoke_super_trampoline_with_access_check, artInvokeSuperTrampolineWithAccessCheck 501INVOKE_TRAMPOLINE art_quick_invoke_virtual_trampoline_with_access_check, artInvokeVirtualTrampolineWithAccessCheck 502 503 504.macro INVOKE_STUB_CREATE_FRAME 505 506SAVE_SIZE=6*8 // x4, x5, x19(wSUSPEND), SP, LR & FP saved. 507SAVE_SIZE_AND_METHOD=SAVE_SIZE+STACK_REFERENCE_SIZE 508 509 510 mov x9, sp // Save stack pointer. 511 .cfi_register sp,x9 512 513 add x10, x2, # SAVE_SIZE_AND_METHOD // calculate size of frame. 514 sub x10, sp, x10 // Calculate SP position - saves + ArtMethod* + args 515 and x10, x10, # ~0xf // Enforce 16 byte stack alignment. 516 mov sp, x10 // Set new SP. 517 518 sub x10, x9, #SAVE_SIZE // Calculate new FP (later). Done here as we must move SP 519 .cfi_def_cfa_register x10 // before this. 520 .cfi_adjust_cfa_offset SAVE_SIZE 521 522 stp x9, x19, [x10, #32] // Save old stack pointer and x19(wSUSPEND) 523 .cfi_rel_offset sp, 32 524 .cfi_rel_offset x19, 40 525 526 stp x4, x5, [x10, #16] // Save result and shorty addresses. 527 .cfi_rel_offset x4, 16 528 .cfi_rel_offset x5, 24 529 530 stp xFP, xLR, [x10] // Store LR & FP. 531 .cfi_rel_offset x29, 0 532 .cfi_rel_offset x30, 8 533 534 mov xFP, x10 // Use xFP now, as it's callee-saved. 535 .cfi_def_cfa_register x29 536 mov xSELF, x3 // Move thread pointer into SELF register. 537 mov wSUSPEND, #SUSPEND_CHECK_INTERVAL // reset wSUSPEND to suspend check interval 538 539 // Copy arguments into stack frame. 540 // Use simple copy routine for now. 541 // 4 bytes per slot. 542 // X1 - source address 543 // W2 - args length 544 // X9 - destination address. 545 // W10 - temporary 546 add x9, sp, #4 // Destination address is bottom of stack + NULL. 547 548 // Use \@ to differentiate between macro invocations. 549.LcopyParams\@: 550 cmp w2, #0 551 beq .LendCopyParams\@ 552 sub w2, w2, #4 // Need 65536 bytes of range. 553 ldr w10, [x1, x2] 554 str w10, [x9, x2] 555 556 b .LcopyParams\@ 557 558.LendCopyParams\@: 559 560 // Store NULL into StackReference<Method>* at bottom of frame. 561 str wzr, [sp] 562 563#if (STACK_REFERENCE_SIZE != 4) 564#error "STACK_REFERENCE_SIZE(ARM64) size not as expected." 565#endif 566.endm 567 568.macro INVOKE_STUB_CALL_AND_RETURN 569 570 // load method-> METHOD_QUICK_CODE_OFFSET 571 ldr x9, [x0 , #METHOD_QUICK_CODE_OFFSET] 572 // Branch to method. 573 blr x9 574 575 // Restore return value address and shorty address. 576 ldp x4,x5, [xFP, #16] 577 .cfi_restore x4 578 .cfi_restore x5 579 580 // Store result (w0/x0/s0/d0) appropriately, depending on resultType. 581 ldrb w10, [x5] 582 583 // Don't set anything for a void type. 584 cmp w10, #'V' 585 beq .Lexit_art_quick_invoke_stub\@ 586 587 cmp w10, #'D' 588 bne .Lreturn_is_float\@ 589 str d0, [x4] 590 b .Lexit_art_quick_invoke_stub\@ 591 592.Lreturn_is_float\@: 593 cmp w10, #'F' 594 bne .Lreturn_is_int\@ 595 str s0, [x4] 596 b .Lexit_art_quick_invoke_stub\@ 597 598 // Just store x0. Doesn't matter if it is 64 or 32 bits. 599.Lreturn_is_int\@: 600 str x0, [x4] 601 602.Lexit_art_quick_invoke_stub\@: 603 ldp x2, x19, [x29, #32] // Restore stack pointer and x19. 604 .cfi_restore x19 605 mov sp, x2 606 .cfi_restore sp 607 608 ldp x29, x30, [x29] // Restore old frame pointer and link register. 609 .cfi_restore x29 610 .cfi_restore x30 611 612 ret 613 614.endm 615 616 617/* 618 * extern"C" void art_quick_invoke_stub(ArtMethod *method, x0 619 * uint32_t *args, x1 620 * uint32_t argsize, w2 621 * Thread *self, x3 622 * JValue *result, x4 623 * char *shorty); x5 624 * +----------------------+ 625 * | | 626 * | C/C++ frame | 627 * | LR'' | 628 * | FP'' | <- SP' 629 * +----------------------+ 630 * +----------------------+ 631 * | SP' | 632 * | X5 | 633 * | X4 | Saved registers 634 * | LR' | 635 * | FP' | <- FP 636 * +----------------------+ 637 * | uint32_t out[n-1] | 638 * | : : | Outs 639 * | uint32_t out[0] | 640 * | StackRef<ArtMethod> | <- SP value=null 641 * +----------------------+ 642 * 643 * Outgoing registers: 644 * x0 - Method* 645 * x1-x7 - integer parameters. 646 * d0-d7 - Floating point parameters. 647 * xSELF = self 648 * wSUSPEND = suspend count 649 * SP = & of ArtMethod* 650 * x1 = "this" pointer. 651 * 652 */ 653ENTRY art_quick_invoke_stub 654 // Spill registers as per AACPS64 calling convention. 655 INVOKE_STUB_CREATE_FRAME 656 657 // Fill registers x/w1 to x/w7 and s/d0 to s/d7 with parameters. 658 // Parse the passed shorty to determine which register to load. 659 // Load addresses for routines that load WXSD registers. 660 adr x11, .LstoreW2 661 adr x12, .LstoreX2 662 adr x13, .LstoreS0 663 adr x14, .LstoreD0 664 665 // Initialize routine offsets to 0 for integers and floats. 666 // x8 for integers, x15 for floating point. 667 mov x8, #0 668 mov x15, #0 669 670 add x10, x5, #1 // Load shorty address, plus one to skip return value. 671 ldr w1, [x9],#4 // Load "this" parameter, and increment arg pointer. 672 673 // Loop to fill registers. 674.LfillRegisters: 675 ldrb w17, [x10], #1 // Load next character in signature, and increment. 676 cbz w17, .LcallFunction // Exit at end of signature. Shorty 0 terminated. 677 678 cmp w17, #'F' // is this a float? 679 bne .LisDouble 680 681 cmp x15, # 8*12 // Skip this load if all registers full. 682 beq .Ladvance4 683 684 add x17, x13, x15 // Calculate subroutine to jump to. 685 br x17 686 687.LisDouble: 688 cmp w17, #'D' // is this a double? 689 bne .LisLong 690 691 cmp x15, # 8*12 // Skip this load if all registers full. 692 beq .Ladvance8 693 694 add x17, x14, x15 // Calculate subroutine to jump to. 695 br x17 696 697.LisLong: 698 cmp w17, #'J' // is this a long? 699 bne .LisOther 700 701 cmp x8, # 6*12 // Skip this load if all registers full. 702 beq .Ladvance8 703 704 add x17, x12, x8 // Calculate subroutine to jump to. 705 br x17 706 707.LisOther: // Everything else takes one vReg. 708 cmp x8, # 6*12 // Skip this load if all registers full. 709 beq .Ladvance4 710 711 add x17, x11, x8 // Calculate subroutine to jump to. 712 br x17 713 714.Ladvance4: 715 add x9, x9, #4 716 b .LfillRegisters 717 718.Ladvance8: 719 add x9, x9, #8 720 b .LfillRegisters 721 722// Macro for loading a parameter into a register. 723// counter - the register with offset into these tables 724// size - the size of the register - 4 or 8 bytes. 725// register - the name of the register to be loaded. 726.macro LOADREG counter size register return 727 ldr \register , [x9], #\size 728 add \counter, \counter, 12 729 b \return 730.endm 731 732// Store ints. 733.LstoreW2: 734 LOADREG x8 4 w2 .LfillRegisters 735 LOADREG x8 4 w3 .LfillRegisters 736 LOADREG x8 4 w4 .LfillRegisters 737 LOADREG x8 4 w5 .LfillRegisters 738 LOADREG x8 4 w6 .LfillRegisters 739 LOADREG x8 4 w7 .LfillRegisters 740 741// Store longs. 742.LstoreX2: 743 LOADREG x8 8 x2 .LfillRegisters 744 LOADREG x8 8 x3 .LfillRegisters 745 LOADREG x8 8 x4 .LfillRegisters 746 LOADREG x8 8 x5 .LfillRegisters 747 LOADREG x8 8 x6 .LfillRegisters 748 LOADREG x8 8 x7 .LfillRegisters 749 750// Store singles. 751.LstoreS0: 752 LOADREG x15 4 s0 .LfillRegisters 753 LOADREG x15 4 s1 .LfillRegisters 754 LOADREG x15 4 s2 .LfillRegisters 755 LOADREG x15 4 s3 .LfillRegisters 756 LOADREG x15 4 s4 .LfillRegisters 757 LOADREG x15 4 s5 .LfillRegisters 758 LOADREG x15 4 s6 .LfillRegisters 759 LOADREG x15 4 s7 .LfillRegisters 760 761// Store doubles. 762.LstoreD0: 763 LOADREG x15 8 d0 .LfillRegisters 764 LOADREG x15 8 d1 .LfillRegisters 765 LOADREG x15 8 d2 .LfillRegisters 766 LOADREG x15 8 d3 .LfillRegisters 767 LOADREG x15 8 d4 .LfillRegisters 768 LOADREG x15 8 d5 .LfillRegisters 769 LOADREG x15 8 d6 .LfillRegisters 770 LOADREG x15 8 d7 .LfillRegisters 771 772 773.LcallFunction: 774 775 INVOKE_STUB_CALL_AND_RETURN 776 777END art_quick_invoke_stub 778 779/* extern"C" 780 * void art_quick_invoke_static_stub(ArtMethod *method, x0 781 * uint32_t *args, x1 782 * uint32_t argsize, w2 783 * Thread *self, x3 784 * JValue *result, x4 785 * char *shorty); x5 786 */ 787ENTRY art_quick_invoke_static_stub 788 // Spill registers as per AACPS64 calling convention. 789 INVOKE_STUB_CREATE_FRAME 790 791 // Fill registers x/w1 to x/w7 and s/d0 to s/d7 with parameters. 792 // Parse the passed shorty to determine which register to load. 793 // Load addresses for routines that load WXSD registers. 794 adr x11, .LstoreW1_2 795 adr x12, .LstoreX1_2 796 adr x13, .LstoreS0_2 797 adr x14, .LstoreD0_2 798 799 // Initialize routine offsets to 0 for integers and floats. 800 // x8 for integers, x15 for floating point. 801 mov x8, #0 802 mov x15, #0 803 804 add x10, x5, #1 // Load shorty address, plus one to skip return value. 805 806 // Loop to fill registers. 807.LfillRegisters2: 808 ldrb w17, [x10], #1 // Load next character in signature, and increment. 809 cbz w17, .LcallFunction2 // Exit at end of signature. Shorty 0 terminated. 810 811 cmp w17, #'F' // is this a float? 812 bne .LisDouble2 813 814 cmp x15, # 8*12 // Skip this load if all registers full. 815 beq .Ladvance4_2 816 817 add x17, x13, x15 // Calculate subroutine to jump to. 818 br x17 819 820.LisDouble2: 821 cmp w17, #'D' // is this a double? 822 bne .LisLong2 823 824 cmp x15, # 8*12 // Skip this load if all registers full. 825 beq .Ladvance8_2 826 827 add x17, x14, x15 // Calculate subroutine to jump to. 828 br x17 829 830.LisLong2: 831 cmp w17, #'J' // is this a long? 832 bne .LisOther2 833 834 cmp x8, # 7*12 // Skip this load if all registers full. 835 beq .Ladvance8_2 836 837 add x17, x12, x8 // Calculate subroutine to jump to. 838 br x17 839 840.LisOther2: // Everything else takes one vReg. 841 cmp x8, # 7*12 // Skip this load if all registers full. 842 beq .Ladvance4_2 843 844 add x17, x11, x8 // Calculate subroutine to jump to. 845 br x17 846 847.Ladvance4_2: 848 add x9, x9, #4 849 b .LfillRegisters2 850 851.Ladvance8_2: 852 add x9, x9, #8 853 b .LfillRegisters2 854 855// Store ints. 856.LstoreW1_2: 857 LOADREG x8 4 w1 .LfillRegisters2 858 LOADREG x8 4 w2 .LfillRegisters2 859 LOADREG x8 4 w3 .LfillRegisters2 860 LOADREG x8 4 w4 .LfillRegisters2 861 LOADREG x8 4 w5 .LfillRegisters2 862 LOADREG x8 4 w6 .LfillRegisters2 863 LOADREG x8 4 w7 .LfillRegisters2 864 865// Store longs. 866.LstoreX1_2: 867 LOADREG x8 8 x1 .LfillRegisters2 868 LOADREG x8 8 x2 .LfillRegisters2 869 LOADREG x8 8 x3 .LfillRegisters2 870 LOADREG x8 8 x4 .LfillRegisters2 871 LOADREG x8 8 x5 .LfillRegisters2 872 LOADREG x8 8 x6 .LfillRegisters2 873 LOADREG x8 8 x7 .LfillRegisters2 874 875// Store singles. 876.LstoreS0_2: 877 LOADREG x15 4 s0 .LfillRegisters2 878 LOADREG x15 4 s1 .LfillRegisters2 879 LOADREG x15 4 s2 .LfillRegisters2 880 LOADREG x15 4 s3 .LfillRegisters2 881 LOADREG x15 4 s4 .LfillRegisters2 882 LOADREG x15 4 s5 .LfillRegisters2 883 LOADREG x15 4 s6 .LfillRegisters2 884 LOADREG x15 4 s7 .LfillRegisters2 885 886// Store doubles. 887.LstoreD0_2: 888 LOADREG x15 8 d0 .LfillRegisters2 889 LOADREG x15 8 d1 .LfillRegisters2 890 LOADREG x15 8 d2 .LfillRegisters2 891 LOADREG x15 8 d3 .LfillRegisters2 892 LOADREG x15 8 d4 .LfillRegisters2 893 LOADREG x15 8 d5 .LfillRegisters2 894 LOADREG x15 8 d6 .LfillRegisters2 895 LOADREG x15 8 d7 .LfillRegisters2 896 897 898.LcallFunction2: 899 900 INVOKE_STUB_CALL_AND_RETURN 901 902END art_quick_invoke_static_stub 903 904 905 906 /* 907 * On entry x0 is uintptr_t* gprs_ and x1 is uint64_t* fprs_ 908 */ 909 910ENTRY art_quick_do_long_jump 911 // Load FPRs 912 ldp d0, d1, [x1], #16 913 ldp d2, d3, [x1], #16 914 ldp d4, d5, [x1], #16 915 ldp d6, d7, [x1], #16 916 ldp d8, d9, [x1], #16 917 ldp d10, d11, [x1], #16 918 ldp d12, d13, [x1], #16 919 ldp d14, d15, [x1], #16 920 ldp d16, d17, [x1], #16 921 ldp d18, d19, [x1], #16 922 ldp d20, d21, [x1], #16 923 ldp d22, d23, [x1], #16 924 ldp d24, d25, [x1], #16 925 ldp d26, d27, [x1], #16 926 ldp d28, d29, [x1], #16 927 ldp d30, d31, [x1] 928 929 // Load GPRs 930 // TODO: lots of those are smashed, could optimize. 931 add x0, x0, #30*8 932 ldp x30, x1, [x0], #-16 933 ldp x28, x29, [x0], #-16 934 ldp x26, x27, [x0], #-16 935 ldp x24, x25, [x0], #-16 936 ldp x22, x23, [x0], #-16 937 ldp x20, x21, [x0], #-16 938 ldp x18, x19, [x0], #-16 939 ldp x16, x17, [x0], #-16 940 ldp x14, x15, [x0], #-16 941 ldp x12, x13, [x0], #-16 942 ldp x10, x11, [x0], #-16 943 ldp x8, x9, [x0], #-16 944 ldp x6, x7, [x0], #-16 945 ldp x4, x5, [x0], #-16 946 ldp x2, x3, [x0], #-16 947 mov sp, x1 948 949 // TODO: Is it really OK to use LR for the target PC? 950 mov x0, #0 951 mov x1, #0 952 br xLR 953END art_quick_do_long_jump 954 955 /* 956 * Entry from managed code that calls artHandleFillArrayDataFromCode and delivers exception on 957 * failure. 958 */ 959 .extern artHandleFillArrayDataFromCode 960ENTRY art_quick_handle_fill_data 961 SETUP_REF_ONLY_CALLEE_SAVE_FRAME // Save callee saves in case exception allocation triggers GC. 962 mov x2, xSELF // Pass Thread::Current. 963 mov x3, sp // Pass SP. 964 bl artHandleFillArrayDataFromCode // (Array*, const DexFile::Payload*, Thread*, SP) 965 RESTORE_REF_ONLY_CALLEE_SAVE_FRAME 966 RETURN_IF_RESULT_IS_ZERO 967 DELIVER_PENDING_EXCEPTION 968END art_quick_handle_fill_data 969 970 /* 971 * Entry from managed code that calls artLockObjectFromCode, may block for GC. x0 holds the 972 * possibly null object to lock. 973 * 974 * Derived from arm32 code. 975 */ 976 .extern artLockObjectFromCode 977ENTRY art_quick_lock_object 978 cbz w0, .Lslow_lock 979 add x4, x0, #LOCK_WORD_OFFSET // exclusive load/store had no immediate anymore 980.Lretry_lock: 981 ldr w2, [xSELF, #THREAD_ID_OFFSET] // TODO: Can the thread ID really change during the loop? 982 ldxr w1, [x4] 983 cbnz w1, .Lnot_unlocked // already thin locked 984 stxr w3, w2, [x4] 985 cbnz w3, .Lstrex_fail // store failed, retry 986 dmb ishld // full (LoadLoad|LoadStore) memory barrier 987 ret 988.Lstrex_fail: 989 b .Lretry_lock // unlikely forward branch, need to reload and recheck r1/r2 990.Lnot_unlocked: 991 lsr w3, w1, 30 992 cbnz w3, .Lslow_lock // if either of the top two bits are set, go slow path 993 eor w2, w1, w2 // lock_word.ThreadId() ^ self->ThreadId() 994 uxth w2, w2 // zero top 16 bits 995 cbnz w2, .Lslow_lock // lock word and self thread id's match -> recursive lock 996 // else contention, go to slow path 997 add w2, w1, #65536 // increment count in lock word placing in w2 for storing 998 lsr w1, w2, 30 // if either of the top two bits are set, we overflowed. 999 cbnz w1, .Lslow_lock // if we overflow the count go slow path 1000 str w2, [x0, #LOCK_WORD_OFFSET]// no need for stxr as we hold the lock 1001 ret 1002.Lslow_lock: 1003 SETUP_REF_ONLY_CALLEE_SAVE_FRAME // save callee saves in case we block 1004 mov x1, xSELF // pass Thread::Current 1005 mov x2, sp // pass SP 1006 bl artLockObjectFromCode // (Object* obj, Thread*, SP) 1007 RESTORE_REF_ONLY_CALLEE_SAVE_FRAME 1008 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1009END art_quick_lock_object 1010 1011 /* 1012 * Entry from managed code that calls artUnlockObjectFromCode and delivers exception on failure. 1013 * x0 holds the possibly null object to lock. 1014 * 1015 * Derived from arm32 code. 1016 */ 1017 .extern artUnlockObjectFromCode 1018ENTRY art_quick_unlock_object 1019 cbz x0, .Lslow_unlock 1020 ldr w1, [x0, #LOCK_WORD_OFFSET] 1021 lsr w2, w1, 30 1022 cbnz w2, .Lslow_unlock // if either of the top two bits are set, go slow path 1023 ldr w2, [xSELF, #THREAD_ID_OFFSET] 1024 eor w3, w1, w2 // lock_word.ThreadId() ^ self->ThreadId() 1025 uxth w3, w3 // zero top 16 bits 1026 cbnz w3, .Lslow_unlock // do lock word and self thread id's match? 1027 cmp w1, #65536 1028 bpl .Lrecursive_thin_unlock 1029 // transition to unlocked, w3 holds 0 1030 dmb ish // full (LoadStore|StoreStore) memory barrier 1031 str w3, [x0, #LOCK_WORD_OFFSET] 1032 ret 1033.Lrecursive_thin_unlock: 1034 sub w1, w1, #65536 1035 str w1, [x0, #LOCK_WORD_OFFSET] 1036 ret 1037.Lslow_unlock: 1038 SETUP_REF_ONLY_CALLEE_SAVE_FRAME // save callee saves in case exception allocation triggers GC 1039 mov x1, xSELF // pass Thread::Current 1040 mov x2, sp // pass SP 1041 bl artUnlockObjectFromCode // (Object* obj, Thread*, SP) 1042 RESTORE_REF_ONLY_CALLEE_SAVE_FRAME 1043 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1044END art_quick_unlock_object 1045 1046 /* 1047 * Entry from managed code that calls artIsAssignableFromCode and on failure calls 1048 * artThrowClassCastException. 1049 */ 1050 .extern artThrowClassCastException 1051ENTRY art_quick_check_cast 1052 // Store arguments and link register 1053 sub sp, sp, #32 // Stack needs to be 16b aligned on calls 1054 .cfi_adjust_cfa_offset 32 1055 stp x0, x1, [sp] 1056 .cfi_rel_offset x0, 0 1057 .cfi_rel_offset x1, 8 1058 stp xSELF, xLR, [sp, #16] 1059 .cfi_rel_offset x18, 16 1060 .cfi_rel_offset x30, 24 1061 1062 // Call runtime code 1063 bl artIsAssignableFromCode 1064 1065 // Check for exception 1066 cbz x0, .Lthrow_class_cast_exception 1067 1068 // Restore and return 1069 ldp x0, x1, [sp] 1070 .cfi_restore x0 1071 .cfi_restore x1 1072 ldp xSELF, xLR, [sp, #16] 1073 .cfi_restore x18 1074 .cfi_restore x30 1075 add sp, sp, #32 1076 .cfi_adjust_cfa_offset -32 1077 ret 1078 1079.Lthrow_class_cast_exception: 1080 // Restore 1081 ldp x0, x1, [sp] 1082 .cfi_restore x0 1083 .cfi_restore x1 1084 ldp xSELF, xLR, [sp, #16] 1085 .cfi_restore x18 1086 .cfi_restore x30 1087 add sp, sp, #32 1088 .cfi_adjust_cfa_offset -32 1089 1090 SETUP_SAVE_ALL_CALLEE_SAVE_FRAME // save all registers as basis for long jump context 1091 mov x2, xSELF // pass Thread::Current 1092 mov x3, sp // pass SP 1093 b artThrowClassCastException // (Class*, Class*, Thread*, SP) 1094 brk 0 // We should not return here... 1095END art_quick_check_cast 1096 1097 /* 1098 * Entry from managed code for array put operations of objects where the value being stored 1099 * needs to be checked for compatibility. 1100 * x0 = array, x1 = index, x2 = value 1101 * 1102 * Currently all values should fit into w0/w1/w2, and w1 always will as indices are 32b. We 1103 * assume, though, that the upper 32b are zeroed out. At least for x1/w1 we can do better by 1104 * using index-zero-extension in load/stores. 1105 * 1106 * Temporaries: x3, x4 1107 * TODO: x4 OK? ip seems wrong here. 1108 */ 1109ENTRY art_quick_aput_obj_with_null_and_bound_check 1110 tst x0, x0 1111 bne art_quick_aput_obj_with_bound_check 1112 b art_quick_throw_null_pointer_exception 1113END art_quick_aput_obj_with_null_and_bound_check 1114 1115ENTRY art_quick_aput_obj_with_bound_check 1116 ldr w3, [x0, #ARRAY_LENGTH_OFFSET] 1117 cmp w3, w1 1118 bhi art_quick_aput_obj 1119 mov x0, x1 1120 mov x1, x3 1121 b art_quick_throw_array_bounds 1122END art_quick_aput_obj_with_bound_check 1123 1124ENTRY art_quick_aput_obj 1125 cbz x2, .Ldo_aput_null 1126 ldr w3, [x0, #CLASS_OFFSET] // Heap reference = 32b 1127 // This also zero-extends to x3 1128 ldr w4, [x2, #CLASS_OFFSET] // Heap reference = 32b 1129 // This also zero-extends to x4 1130 ldr w3, [x3, #CLASS_COMPONENT_TYPE_OFFSET] // Heap reference = 32b 1131 // This also zero-extends to x3 1132 cmp w3, w4 // value's type == array's component type - trivial assignability 1133 bne .Lcheck_assignability 1134.Ldo_aput: 1135 add x3, x0, #OBJECT_ARRAY_DATA_OFFSET 1136 // "Compress" = do nothing 1137 str w2, [x3, x1, lsl #2] // Heap reference = 32b 1138 ldr x3, [xSELF, #THREAD_CARD_TABLE_OFFSET] 1139 lsr x0, x0, #7 1140 strb w3, [x3, x0] 1141 ret 1142.Ldo_aput_null: 1143 add x3, x0, #OBJECT_ARRAY_DATA_OFFSET 1144 // "Compress" = do nothing 1145 str w2, [x3, x1, lsl #2] // Heap reference = 32b 1146 ret 1147.Lcheck_assignability: 1148 // Store arguments and link register 1149 sub sp, sp, #48 // Stack needs to be 16b aligned on calls 1150 .cfi_adjust_cfa_offset 48 1151 stp x0, x1, [sp] 1152 .cfi_rel_offset x0, 0 1153 .cfi_rel_offset x1, 8 1154 stp x2, xSELF, [sp, #16] 1155 .cfi_rel_offset x2, 16 1156 .cfi_rel_offset x18, 24 1157 str xLR, [sp, #32] 1158 .cfi_rel_offset x30, 32 1159 1160 // Call runtime code 1161 mov x0, x3 // Heap reference, 32b, "uncompress" = do nothing, already zero-extended 1162 mov x1, x4 // Heap reference, 32b, "uncompress" = do nothing, already zero-extended 1163 bl artIsAssignableFromCode 1164 1165 // Check for exception 1166 cbz x0, .Lthrow_array_store_exception 1167 1168 // Restore 1169 ldp x0, x1, [sp] 1170 .cfi_restore x0 1171 .cfi_restore x1 1172 ldp x2, xSELF, [sp, #16] 1173 .cfi_restore x2 1174 .cfi_restore x18 1175 ldr xLR, [sp, #32] 1176 .cfi_restore x30 1177 add sp, sp, #48 1178 .cfi_adjust_cfa_offset -48 1179 1180 add x3, x0, #OBJECT_ARRAY_DATA_OFFSET 1181 // "Compress" = do nothing 1182 str w2, [x3, x1, lsl #2] // Heap reference = 32b 1183 ldr x3, [xSELF, #THREAD_CARD_TABLE_OFFSET] 1184 lsr x0, x0, #7 1185 strb w3, [x3, x0] 1186 ret 1187.Lthrow_array_store_exception: 1188 ldp x0, x1, [sp] 1189 .cfi_restore x0 1190 .cfi_restore x1 1191 ldp x2, xSELF, [sp, #16] 1192 .cfi_restore x2 1193 .cfi_restore x18 1194 ldr xLR, [sp, #32] 1195 .cfi_restore x30 1196 add sp, sp, #48 1197 .cfi_adjust_cfa_offset -48 1198 1199 SETUP_SAVE_ALL_CALLEE_SAVE_FRAME 1200 mov x1, x2 // Pass value. 1201 mov x2, xSELF // Pass Thread::Current. 1202 mov x3, sp // Pass SP. 1203 b artThrowArrayStoreException // (Object*, Object*, Thread*, SP). 1204 brk 0 // Unreached. 1205END art_quick_aput_obj 1206 1207// Macro to facilitate adding new allocation entrypoints. 1208.macro TWO_ARG_DOWNCALL name, entrypoint, return 1209 .extern \entrypoint 1210ENTRY \name 1211 SETUP_REF_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1212 mov x2, xSELF // pass Thread::Current 1213 mov x3, sp // pass SP 1214 bl \entrypoint // (uint32_t type_idx, Method* method, Thread*, SP) 1215 RESTORE_REF_ONLY_CALLEE_SAVE_FRAME 1216 \return 1217 DELIVER_PENDING_EXCEPTION 1218END \name 1219.endm 1220 1221// Macro to facilitate adding new array allocation entrypoints. 1222.macro THREE_ARG_DOWNCALL name, entrypoint, return 1223 .extern \entrypoint 1224ENTRY \name 1225 SETUP_REF_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1226 mov x3, xSELF // pass Thread::Current 1227 mov x4, sp // pass SP 1228 bl \entrypoint 1229 RESTORE_REF_ONLY_CALLEE_SAVE_FRAME 1230 \return 1231 DELIVER_PENDING_EXCEPTION 1232END \name 1233.endm 1234 1235// Macros taking opportunity of code similarities for downcalls with referrer. 1236 1237// TODO: xSELF -> x19. Temporarily rely on xSELF being saved in REF_ONLY 1238.macro ONE_ARG_REF_DOWNCALL name, entrypoint, return 1239 .extern \entrypoint 1240ENTRY \name 1241 SETUP_REF_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1242 ldr w1, [sp, #FRAME_SIZE_REFS_ONLY_CALLEE_SAVE] // Load referrer 1243 mov x2, xSELF // pass Thread::Current 1244 mov x3, sp // pass SP 1245 bl \entrypoint // (uint32_t type_idx, Method* method, Thread*, SP) 1246 RESTORE_REF_ONLY_CALLEE_SAVE_FRAME 1247 \return 1248END \name 1249.endm 1250 1251// TODO: xSELF -> x19. Temporarily rely on xSELF being saved in REF_ONLY 1252.macro TWO_ARG_REF_DOWNCALL name, entrypoint, return 1253 .extern \entrypoint 1254ENTRY \name 1255 SETUP_REF_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1256 ldr w2, [sp, #FRAME_SIZE_REFS_ONLY_CALLEE_SAVE] // Load referrer 1257 mov x3, xSELF // pass Thread::Current 1258 mov x4, sp // pass SP 1259 bl \entrypoint 1260 RESTORE_REF_ONLY_CALLEE_SAVE_FRAME 1261 \return 1262END \name 1263.endm 1264 1265// TODO: xSELF -> x19. Temporarily rely on xSELF being saved in REF_ONLY 1266.macro THREE_ARG_REF_DOWNCALL name, entrypoint, return 1267 .extern \entrypoint 1268ENTRY \name 1269 SETUP_REF_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1270 ldr w3, [sp, #FRAME_SIZE_REFS_ONLY_CALLEE_SAVE] // Load referrer 1271 mov x4, xSELF // pass Thread::Current 1272 mov x5, sp // pass SP 1273 bl \entrypoint 1274 RESTORE_REF_ONLY_CALLEE_SAVE_FRAME 1275 \return 1276END \name 1277.endm 1278 1279 /* 1280 * Entry from managed code when uninitialized static storage, this stub will run the class 1281 * initializer and deliver the exception on error. On success the static storage base is 1282 * returned. 1283 */ 1284TWO_ARG_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode, RETURN_IF_RESULT_IS_NON_ZERO 1285 1286UNIMPLEMENTED art_quick_initialize_type 1287UNIMPLEMENTED art_quick_initialize_type_and_verify_access 1288 1289ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1290ONE_ARG_REF_DOWNCALL art_quick_get64_static, artGet64StaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1291ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1292 1293TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1294TWO_ARG_REF_DOWNCALL art_quick_get64_instance, artGet64InstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1295TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1296 1297TWO_ARG_REF_DOWNCALL art_quick_set32_static, artSet32StaticFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1298TWO_ARG_REF_DOWNCALL art_quick_set_obj_static, artSetObjStaticFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1299 1300THREE_ARG_REF_DOWNCALL art_quick_set32_instance, artSet32InstanceFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1301THREE_ARG_DOWNCALL art_quick_set64_instance, artSet64InstanceFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1302THREE_ARG_REF_DOWNCALL art_quick_set_obj_instance, artSetObjInstanceFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1303 1304// This is separated out as the argument order is different. 1305 .extern artSet64StaticFromCode 1306ENTRY art_quick_set64_static 1307 SETUP_REF_ONLY_CALLEE_SAVE_FRAME // save callee saves in case of GC 1308 mov x3, x1 // Store value 1309 ldr w1, [sp, #FRAME_SIZE_REFS_ONLY_CALLEE_SAVE] // Load referrer 1310 mov x2, x3 // Put value param 1311 mov x3, xSELF // pass Thread::Current 1312 mov x4, sp // pass SP 1313 bl artSet64StaticFromCode 1314 RESTORE_REF_ONLY_CALLEE_SAVE_FRAME 1315 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1316END art_quick_set64_static 1317 1318 /* 1319 * Entry from managed code to resolve a string, this stub will allocate a String and deliver an 1320 * exception on error. On success the String is returned. x0 holds the referring method, 1321 * w1 holds the string index. The fast path check for hit in strings cache has already been 1322 * performed. 1323 */ 1324TWO_ARG_DOWNCALL art_quick_resolve_string, artResolveStringFromCode, RETURN_IF_RESULT_IS_NON_ZERO 1325 1326// Generate the allocation entrypoints for each allocator. 1327GENERATE_ALL_ALLOC_ENTRYPOINTS 1328 1329 /* 1330 * Called by managed code when the value in wSUSPEND has been decremented to 0. 1331 */ 1332 .extern artTestSuspendFromCode 1333ENTRY art_quick_test_suspend 1334 ldrh w0, [xSELF, #THREAD_FLAGS_OFFSET] // get xSELF->state_and_flags.as_struct.flags 1335 mov wSUSPEND, #SUSPEND_CHECK_INTERVAL // reset wSUSPEND to SUSPEND_CHECK_INTERVAL 1336 cbnz w0, .Lneed_suspend // check flags == 0 1337 ret // return if flags == 0 1338.Lneed_suspend: 1339 mov x0, xSELF 1340 SETUP_REF_ONLY_CALLEE_SAVE_FRAME // save callee saves for stack crawl 1341 mov x1, sp 1342 bl artTestSuspendFromCode // (Thread*, SP) 1343 RESTORE_REF_ONLY_CALLEE_SAVE_FRAME_AND_RETURN 1344END art_quick_test_suspend 1345 1346 /* 1347 * Called by managed code that is attempting to call a method on a proxy class. On entry 1348 * x0 holds the proxy method and x1 holds the receiver; The frame size of the invoked proxy 1349 * method agrees with a ref and args callee save frame. 1350 */ 1351 .extern artQuickProxyInvokeHandler 1352ENTRY art_quick_proxy_invoke_handler 1353 SETUP_REF_AND_ARGS_CALLEE_SAVE_FRAME 1354 str x0, [sp, #0] // place proxy method at bottom of frame 1355 mov x2, xSELF // pass Thread::Current 1356 mov x3, sp // pass SP 1357 bl artQuickProxyInvokeHandler // (Method* proxy method, receiver, Thread*, SP) 1358 ldr xSELF, [sp, #200] // Restore self pointer. 1359 ldr x2, [xSELF, THREAD_EXCEPTION_OFFSET] 1360 cbnz x2, .Lexception_in_proxy // success if no exception is pending 1361 RESTORE_REF_AND_ARGS_CALLEE_SAVE_FRAME // Restore frame 1362 fmov d0, x0 // Store result in d0 in case it was float or double 1363 ret // return on success 1364.Lexception_in_proxy: 1365 RESTORE_REF_AND_ARGS_CALLEE_SAVE_FRAME 1366 DELIVER_PENDING_EXCEPTION 1367END art_quick_proxy_invoke_handler 1368 1369 /* 1370 * Called to resolve an imt conflict. x12 is a hidden argument that holds the target method's 1371 * dex method index. 1372 */ 1373ENTRY art_quick_imt_conflict_trampoline 1374 ldr w0, [sp, #0] // load caller Method* 1375 ldr w0, [x0, #METHOD_DEX_CACHE_METHODS_OFFSET] // load dex_cache_resolved_methods 1376 add x0, x0, #OBJECT_ARRAY_DATA_OFFSET // get starting address of data 1377 ldr w0, [x0, x12, lsl 2] // load the target method 1378 b art_quick_invoke_interface_trampoline 1379END art_quick_imt_conflict_trampoline 1380 1381ENTRY art_quick_resolution_trampoline 1382 SETUP_REF_AND_ARGS_CALLEE_SAVE_FRAME 1383 mov x2, xSELF 1384 mov x3, sp 1385 bl artQuickResolutionTrampoline // (called, receiver, Thread*, SP) 1386 cbz x0, 1f 1387 mov x9, x0 // Remember returned code pointer in x9. 1388 ldr w0, [sp, #0] // artQuickResolutionTrampoline puts called method in *SP. 1389 RESTORE_REF_AND_ARGS_CALLEE_SAVE_FRAME 1390 br x9 13911: 1392 RESTORE_REF_AND_ARGS_CALLEE_SAVE_FRAME 1393 DELIVER_PENDING_EXCEPTION 1394END art_quick_resolution_trampoline 1395 1396/* 1397 * Generic JNI frame layout: 1398 * 1399 * #-------------------# 1400 * | | 1401 * | caller method... | 1402 * #-------------------# <--- SP on entry 1403 * | Return X30/LR | 1404 * | X29/FP | callee save 1405 * | X28 | callee save 1406 * | X27 | callee save 1407 * | X26 | callee save 1408 * | X25 | callee save 1409 * | X24 | callee save 1410 * | X23 | callee save 1411 * | X22 | callee save 1412 * | X21 | callee save 1413 * | X20 | callee save 1414 * | X19 | callee save 1415 * | X7 | arg7 1416 * | X6 | arg6 1417 * | X5 | arg5 1418 * | X4 | arg4 1419 * | X3 | arg3 1420 * | X2 | arg2 1421 * | X1 | arg1 1422 * | D15 | float arg 8 1423 * | D14 | float arg 8 1424 * | D13 | float arg 8 1425 * | D12 | callee save 1426 * | D11 | callee save 1427 * | D10 | callee save 1428 * | D9 | callee save 1429 * | D8 | callee save 1430 * | D7 | float arg 8 1431 * | D6 | float arg 7 1432 * | D5 | float arg 6 1433 * | D4 | float arg 5 1434 * | D3 | float arg 4 1435 * | D2 | float arg 3 1436 * | D1 | float arg 2 1437 * | D0 | float arg 1 1438 * | Method* | <- X0 1439 * #-------------------# 1440 * | local ref cookie | // 4B 1441 * | handle scope size | // 4B 1442 * #-------------------# 1443 * | JNI Call Stack | 1444 * #-------------------# <--- SP on native call 1445 * | | 1446 * | Stack for Regs | The trampoline assembly will pop these values 1447 * | | into registers for native call 1448 * #-------------------# 1449 * | Native code ptr | 1450 * #-------------------# 1451 * | Free scratch | 1452 * #-------------------# 1453 * | Ptr to (1) | <--- SP 1454 * #-------------------# 1455 */ 1456 /* 1457 * Called to do a generic JNI down-call 1458 */ 1459ENTRY art_quick_generic_jni_trampoline 1460 SETUP_REF_AND_ARGS_CALLEE_SAVE_FRAME_INTERNAL 1461 str x0, [sp, #0] // Store native ArtMethod* to bottom of stack. 1462 1463 // Save SP , so we can have static CFI info. 1464 mov x28, sp 1465 .cfi_def_cfa_register x28 1466 1467 // This looks the same, but is different: this will be updated to point to the bottom 1468 // of the frame when the handle scope is inserted. 1469 mov xFP, sp 1470 1471 mov x8, #5120 1472 sub sp, sp, x8 1473 1474 // prepare for artQuickGenericJniTrampoline call 1475 // (Thread*, SP) 1476 // x0 x1 <= C calling convention 1477 // xSELF xFP <= where they are 1478 1479 mov x0, xSELF // Thread* 1480 mov x1, xFP 1481 bl artQuickGenericJniTrampoline // (Thread*, sp) 1482 1483 // Get the updated pointer. This is the bottom of the frame _with_ handle scope. 1484 ldr xFP, [sp] 1485 add x9, sp, #8 1486 1487 cmp x0, #0 1488 b.mi .Lentry_error // Check for error, negative value. 1489 1490 // release part of the alloca. 1491 add x9, x9, x0 1492 1493 // Get the code pointer 1494 ldr xIP0, [x9, #0] 1495 1496 // Load parameters from frame into registers. 1497 // TODO Check with artQuickGenericJniTrampoline. 1498 // Also, check again APPCS64 - the stack arguments are interleaved. 1499 ldp x0, x1, [x9, #8] 1500 ldp x2, x3, [x9, #24] 1501 ldp x4, x5, [x9, #40] 1502 ldp x6, x7, [x9, #56] 1503 1504 ldp d0, d1, [x9, #72] 1505 ldp d2, d3, [x9, #88] 1506 ldp d4, d5, [x9, #104] 1507 ldp d6, d7, [x9, #120] 1508 1509 add sp, x9, #136 1510 1511 blr xIP0 // native call. 1512 1513 // Restore self pointer. 1514 ldr xSELF, [x28, #200] 1515 1516 // result sign extension is handled in C code 1517 // prepare for artQuickGenericJniEndTrampoline call 1518 // (Thread*, SP, result, result_f) 1519 // x0 x1 x2 x3 <= C calling convention 1520 mov x5, x0 // Save return value 1521 mov x0, xSELF // Thread register 1522 mov x1, xFP // Stack pointer 1523 mov x2, x5 // Result (from saved) 1524 fmov x3, d0 // d0 will contain floating point result, but needs to go into x3 1525 1526 bl artQuickGenericJniEndTrampoline 1527 1528 // Tear down the alloca. 1529 mov sp, x28 1530 .cfi_def_cfa_register sp 1531 1532 // Restore self pointer. 1533 ldr xSELF, [x28, #200] 1534 1535 // Pending exceptions possible. 1536 ldr x1, [xSELF, THREAD_EXCEPTION_OFFSET] 1537 cbnz x1, .Lexception_in_native 1538 1539 // Tear down the callee-save frame. 1540 RESTORE_REF_AND_ARGS_CALLEE_SAVE_FRAME 1541 1542 // store into fpr, for when it's a fpr return... 1543 fmov d0, x0 1544 ret 1545 1546.Lentry_error: 1547 mov sp, x28 1548 .cfi_def_cfa_register sp 1549 ldr xSELF, [x28, #200] 1550.Lexception_in_native: 1551 RESTORE_REF_AND_ARGS_CALLEE_SAVE_FRAME 1552 DELIVER_PENDING_EXCEPTION 1553 1554END art_quick_generic_jni_trampoline 1555 1556/* 1557 * Called to bridge from the quick to interpreter ABI. On entry the arguments match those 1558 * of a quick call: 1559 * x0 = method being called/to bridge to. 1560 * x1..x7, d0..d7 = arguments to that method. 1561 */ 1562ENTRY art_quick_to_interpreter_bridge 1563 SETUP_REF_AND_ARGS_CALLEE_SAVE_FRAME // Set up frame and save arguments. 1564 1565 // x0 will contain mirror::ArtMethod* method. 1566 mov x1, xSELF // How to get Thread::Current() ??? 1567 mov x2, sp 1568 1569 // uint64_t artQuickToInterpreterBridge(mirror::ArtMethod* method, Thread* self, 1570 // mirror::ArtMethod** sp) 1571 bl artQuickToInterpreterBridge 1572 1573 RESTORE_REF_AND_ARGS_CALLEE_SAVE_FRAME // TODO: no need to restore arguments in this case. 1574 1575 fmov d0, x0 1576 1577 RETURN_OR_DELIVER_PENDING_EXCEPTION 1578END art_quick_to_interpreter_bridge 1579 1580UNIMPLEMENTED art_quick_instrumentation_entry 1581UNIMPLEMENTED art_quick_instrumentation_exit 1582UNIMPLEMENTED art_quick_deoptimize 1583UNIMPLEMENTED art_quick_indexof 1584 1585 /* 1586 * String's compareTo. 1587 * 1588 * TODO: Not very optimized. 1589 * 1590 * On entry: 1591 * x0: this object pointer 1592 * x1: comp object pointer 1593 * 1594 */ 1595 .extern __memcmp16 1596ENTRY art_quick_string_compareto 1597 mov x2, x0 // x0 is return, use x2 for first input. 1598 sub x0, x2, x1 // Same string object? 1599 cbnz x0,1f 1600 ret 16011: // Different string objects. 1602 1603 ldr w6, [x2, #STRING_OFFSET_OFFSET] 1604 ldr w5, [x1, #STRING_OFFSET_OFFSET] 1605 ldr w4, [x2, #STRING_COUNT_OFFSET] 1606 ldr w3, [x1, #STRING_COUNT_OFFSET] 1607 ldr w2, [x2, #STRING_VALUE_OFFSET] 1608 ldr w1, [x1, #STRING_VALUE_OFFSET] 1609 1610 /* 1611 * Now: CharArray* Offset Count 1612 * first arg x2 w6 w4 1613 * second arg x1 w5 w3 1614 */ 1615 1616 // x0 := str1.length(w4) - str2.length(w3). ldr zero-extended w3/w4 into x3/x4. 1617 subs x0, x4, x3 1618 // Min(count1, count2) into w3. 1619 csel x3, x3, x4, ge 1620 1621 // Build pointer into string data. 1622 1623 // Add offset in array (substr etc.) (sign extend and << 1). 1624 add x2, x2, w6, sxtw #1 1625 add x1, x1, w5, sxtw #1 1626 1627 // Add offset in CharArray to array. 1628 add x2, x2, #STRING_DATA_OFFSET 1629 add x1, x1, #STRING_DATA_OFFSET 1630 1631 // Check for long string, do memcmp16 for them. 1632 cmp w3, #28 // Constant from arm32. 1633 bgt .Ldo_memcmp16 1634 1635 /* 1636 * Now: 1637 * x2: *first string data 1638 * x1: *second string data 1639 * w3: iteration count 1640 * x0: return value if comparison equal 1641 * x4, x5, x6, x7: free 1642 */ 1643 1644 // Do a simple unrolled loop. 1645.Lloop: 1646 // At least two more elements? 1647 subs w3, w3, #2 1648 b.lt .Lremainder_or_done 1649 1650 ldrh w4, [x2], #2 1651 ldrh w5, [x1], #2 1652 1653 ldrh w6, [x2], #2 1654 ldrh w7, [x1], #2 1655 1656 subs w4, w4, w5 1657 b.ne .Lw4_result 1658 1659 subs w6, w6, w7 1660 b.ne .Lw6_result 1661 1662 b .Lloop 1663 1664.Lremainder_or_done: 1665 adds w3, w3, #1 1666 b.eq .Lremainder 1667 ret 1668 1669.Lremainder: 1670 ldrh w4, [x2], #2 1671 ldrh w5, [x1], #2 1672 subs w4, w4, w5 1673 b.ne .Lw4_result 1674 ret 1675 1676// Result is in w4 1677.Lw4_result: 1678 sxtw x0, w4 1679 ret 1680 1681// Result is in w6 1682.Lw6_result: 1683 sxtw x0, w6 1684 ret 1685 1686.Ldo_memcmp16: 1687 str x0, [sp,#-16]! // Save x0 1688 1689 mov x0, x2 1690 uxtw x2, w3 1691 bl __memcmp16 1692 1693 ldr x1, [sp], #16 // Restore old x0 = length diff 1694 1695 cmp x0, #0 // Check the memcmp difference 1696 csel x0, x0, x1, ne // x0 := x0 != 0 ? x0 : x1 1697 ret 1698END art_quick_string_compareto 1699