1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "asm_support_arm64.S" 18#include "interpreter/cfi_asm_support.h" 19 20#include "arch/quick_alloc_entrypoints.S" 21 22 23.macro INCREASE_FRAME frame_adjustment 24 sub sp, sp, #(\frame_adjustment) 25 .cfi_adjust_cfa_offset (\frame_adjustment) 26.endm 27 28.macro DECREASE_FRAME frame_adjustment 29 add sp, sp, #(\frame_adjustment) 30 .cfi_adjust_cfa_offset -(\frame_adjustment) 31.endm 32 33.macro SAVE_REG reg, offset 34 str \reg, [sp, #(\offset)] 35 .cfi_rel_offset \reg, (\offset) 36.endm 37 38.macro RESTORE_REG reg, offset 39 ldr \reg, [sp, #(\offset)] 40 .cfi_restore \reg 41.endm 42 43.macro SAVE_REG_INCREASE_FRAME reg, frame_adjustment 44 str \reg, [sp, #-(\frame_adjustment)]! 45 .cfi_adjust_cfa_offset (\frame_adjustment) 46 .cfi_rel_offset \reg, 0 47.endm 48 49.macro RESTORE_REG_DECREASE_FRAME reg, frame_adjustment 50 ldr \reg, [sp], #(\frame_adjustment) 51 .cfi_restore \reg 52 .cfi_adjust_cfa_offset -(\frame_adjustment) 53.endm 54 55.macro SAVE_TWO_REGS reg1, reg2, offset 56 stp \reg1, \reg2, [sp, #(\offset)] 57 .cfi_rel_offset \reg1, (\offset) 58 .cfi_rel_offset \reg2, (\offset) + 8 59.endm 60 61.macro RESTORE_TWO_REGS reg1, reg2, offset 62 ldp \reg1, \reg2, [sp, #(\offset)] 63 .cfi_restore \reg1 64 .cfi_restore \reg2 65.endm 66 67.macro SAVE_TWO_REGS_INCREASE_FRAME reg1, reg2, frame_adjustment 68 stp \reg1, \reg2, [sp, #-(\frame_adjustment)]! 69 .cfi_adjust_cfa_offset (\frame_adjustment) 70 .cfi_rel_offset \reg1, 0 71 .cfi_rel_offset \reg2, 8 72.endm 73 74.macro RESTORE_TWO_REGS_DECREASE_FRAME reg1, reg2, frame_adjustment 75 ldp \reg1, \reg2, [sp], #(\frame_adjustment) 76 .cfi_restore \reg1 77 .cfi_restore \reg2 78 .cfi_adjust_cfa_offset -(\frame_adjustment) 79.endm 80 81 /* 82 * Macro that sets up the callee save frame to conform with 83 * Runtime::CreateCalleeSaveMethod(kSaveAllCalleeSaves) 84 */ 85.macro SETUP_SAVE_ALL_CALLEE_SAVES_FRAME 86 // art::Runtime** xIP0 = &art::Runtime::instance_ 87 adrp xIP0, :got:_ZN3art7Runtime9instance_E 88 ldr xIP0, [xIP0, #:got_lo12:_ZN3art7Runtime9instance_E] 89 90 // Our registers aren't intermixed - just spill in order. 91 ldr xIP0, [xIP0] // art::Runtime* xIP0 = art::Runtime::instance_; 92 93 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveAllCalleeSaves]; 94 ldr xIP0, [xIP0, RUNTIME_SAVE_ALL_CALLEE_SAVES_METHOD_OFFSET] 95 96 INCREASE_FRAME 176 97 98 // Ugly compile-time check, but we only have the preprocessor. 99#if (FRAME_SIZE_SAVE_ALL_CALLEE_SAVES != 176) 100#error "FRAME_SIZE_SAVE_ALL_CALLEE_SAVES(ARM64) size not as expected." 101#endif 102 103 // Stack alignment filler [sp, #8]. 104 // FP callee-saves. 105 stp d8, d9, [sp, #16] 106 stp d10, d11, [sp, #32] 107 stp d12, d13, [sp, #48] 108 stp d14, d15, [sp, #64] 109 110 // GP callee-saves 111 SAVE_TWO_REGS x19, x20, 80 112 SAVE_TWO_REGS x21, x22, 96 113 SAVE_TWO_REGS x23, x24, 112 114 SAVE_TWO_REGS x25, x26, 128 115 SAVE_TWO_REGS x27, x28, 144 116 SAVE_TWO_REGS x29, xLR, 160 117 118 // Store ArtMethod* Runtime::callee_save_methods_[kSaveAllCalleeSaves]. 119 str xIP0, [sp] 120 // Place sp in Thread::Current()->top_quick_frame. 121 mov xIP0, sp 122 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 123.endm 124 125 /* 126 * Macro that sets up the callee save frame to conform with 127 * Runtime::CreateCalleeSaveMethod(kSaveRefsOnly). 128 */ 129.macro SETUP_SAVE_REFS_ONLY_FRAME 130 // art::Runtime** xIP0 = &art::Runtime::instance_ 131 adrp xIP0, :got:_ZN3art7Runtime9instance_E 132 ldr xIP0, [xIP0, #:got_lo12:_ZN3art7Runtime9instance_E] 133 134 // Our registers aren't intermixed - just spill in order. 135 ldr xIP0, [xIP0] // art::Runtime* xIP0 = art::Runtime::instance_; 136 137 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveRefOnly]; 138 ldr xIP0, [xIP0, RUNTIME_SAVE_REFS_ONLY_METHOD_OFFSET] 139 140 INCREASE_FRAME 96 141 142 // Ugly compile-time check, but we only have the preprocessor. 143#if (FRAME_SIZE_SAVE_REFS_ONLY != 96) 144#error "FRAME_SIZE_SAVE_REFS_ONLY(ARM64) size not as expected." 145#endif 146 147 // GP callee-saves. 148 // x20 paired with ArtMethod* - see below. 149 SAVE_TWO_REGS x21, x22, 16 150 SAVE_TWO_REGS x23, x24, 32 151 SAVE_TWO_REGS x25, x26, 48 152 SAVE_TWO_REGS x27, x28, 64 153 SAVE_TWO_REGS x29, xLR, 80 154 155 // Store ArtMethod* Runtime::callee_save_methods_[kSaveRefsOnly]. 156 // Note: We could avoid saving X20 in the case of Baker read 157 // barriers, as it is overwritten by REFRESH_MARKING_REGISTER 158 // later; but it's not worth handling this special case. 159 stp xIP0, x20, [sp] 160 .cfi_rel_offset x20, 8 161 162 // Place sp in Thread::Current()->top_quick_frame. 163 mov xIP0, sp 164 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 165.endm 166 167// TODO: Probably no need to restore registers preserved by aapcs64. 168.macro RESTORE_SAVE_REFS_ONLY_FRAME 169 // Callee-saves. 170 // Note: Likewise, we could avoid restoring X20 in the case of Baker 171 // read barriers, as it is overwritten by REFRESH_MARKING_REGISTER 172 // later; but it's not worth handling this special case. 173 RESTORE_REG x20, 8 174 RESTORE_TWO_REGS x21, x22, 16 175 RESTORE_TWO_REGS x23, x24, 32 176 RESTORE_TWO_REGS x25, x26, 48 177 RESTORE_TWO_REGS x27, x28, 64 178 RESTORE_TWO_REGS x29, xLR, 80 179 180 DECREASE_FRAME 96 181.endm 182 183.macro POP_SAVE_REFS_ONLY_FRAME 184 DECREASE_FRAME 96 185.endm 186 187 188.macro SETUP_SAVE_REFS_AND_ARGS_FRAME_INTERNAL 189 INCREASE_FRAME 224 190 191 // Ugly compile-time check, but we only have the preprocessor. 192#if (FRAME_SIZE_SAVE_REFS_AND_ARGS != 224) 193#error "FRAME_SIZE_SAVE_REFS_AND_ARGS(ARM64) size not as expected." 194#endif 195 196 // Stack alignment filler [sp, #8]. 197 // FP args. 198 stp d0, d1, [sp, #16] 199 stp d2, d3, [sp, #32] 200 stp d4, d5, [sp, #48] 201 stp d6, d7, [sp, #64] 202 203 // Core args. 204 SAVE_TWO_REGS x1, x2, 80 205 SAVE_TWO_REGS x3, x4, 96 206 SAVE_TWO_REGS x5, x6, 112 207 208 // x7, Callee-saves. 209 // Note: We could avoid saving X20 in the case of Baker read 210 // barriers, as it is overwritten by REFRESH_MARKING_REGISTER 211 // later; but it's not worth handling this special case. 212 SAVE_TWO_REGS x7, x20, 128 213 SAVE_TWO_REGS x21, x22, 144 214 SAVE_TWO_REGS x23, x24, 160 215 SAVE_TWO_REGS x25, x26, 176 216 SAVE_TWO_REGS x27, x28, 192 217 218 // x29(callee-save) and LR. 219 SAVE_TWO_REGS x29, xLR, 208 220 221.endm 222 223 /* 224 * Macro that sets up the callee save frame to conform with 225 * Runtime::CreateCalleeSaveMethod(kSaveRefsAndArgs). 226 * 227 * TODO This is probably too conservative - saving FP & LR. 228 */ 229.macro SETUP_SAVE_REFS_AND_ARGS_FRAME 230 // art::Runtime** xIP0 = &art::Runtime::instance_ 231 adrp xIP0, :got:_ZN3art7Runtime9instance_E 232 ldr xIP0, [xIP0, #:got_lo12:_ZN3art7Runtime9instance_E] 233 234 // Our registers aren't intermixed - just spill in order. 235 ldr xIP0, [xIP0] // art::Runtime* xIP0 = art::Runtime::instance_; 236 237 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveRefAndArgs]; 238 ldr xIP0, [xIP0, RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET] 239 240 SETUP_SAVE_REFS_AND_ARGS_FRAME_INTERNAL 241 242 str xIP0, [sp] // Store ArtMethod* Runtime::callee_save_methods_[kSaveRefsAndArgs]. 243 // Place sp in Thread::Current()->top_quick_frame. 244 mov xIP0, sp 245 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 246.endm 247 248.macro SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_X0 249 SETUP_SAVE_REFS_AND_ARGS_FRAME_INTERNAL 250 str x0, [sp, #0] // Store ArtMethod* to bottom of stack. 251 // Place sp in Thread::Current()->top_quick_frame. 252 mov xIP0, sp 253 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 254.endm 255 256// TODO: Probably no need to restore registers preserved by aapcs64. 257.macro RESTORE_SAVE_REFS_AND_ARGS_FRAME 258 // FP args. 259 ldp d0, d1, [sp, #16] 260 ldp d2, d3, [sp, #32] 261 ldp d4, d5, [sp, #48] 262 ldp d6, d7, [sp, #64] 263 264 // Core args. 265 RESTORE_TWO_REGS x1, x2, 80 266 RESTORE_TWO_REGS x3, x4, 96 267 RESTORE_TWO_REGS x5, x6, 112 268 269 // x7, Callee-saves. 270 // Note: Likewise, we could avoid restoring X20 in the case of Baker 271 // read barriers, as it is overwritten by REFRESH_MARKING_REGISTER 272 // later; but it's not worth handling this special case. 273 RESTORE_TWO_REGS x7, x20, 128 274 RESTORE_TWO_REGS x21, x22, 144 275 RESTORE_TWO_REGS x23, x24, 160 276 RESTORE_TWO_REGS x25, x26, 176 277 RESTORE_TWO_REGS x27, x28, 192 278 279 // x29(callee-save) and LR. 280 RESTORE_TWO_REGS x29, xLR, 208 281 282 DECREASE_FRAME 224 283.endm 284 285 /* 286 * Macro that sets up the callee save frame to conform with 287 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 288 * when the SP has already been decremented by FRAME_SIZE_SAVE_EVERYTHING 289 * and saving registers x29 and LR is handled elsewhere. 290 */ 291.macro SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 292 // Ugly compile-time check, but we only have the preprocessor. 293#if (FRAME_SIZE_SAVE_EVERYTHING != 512) 294#error "FRAME_SIZE_SAVE_EVERYTHING(ARM64) size not as expected." 295#endif 296 297 // Save FP registers. 298 // For better performance, store d0 and d31 separately, so that all STPs are 16-byte aligned. 299 str d0, [sp, #8] 300 stp d1, d2, [sp, #16] 301 stp d3, d4, [sp, #32] 302 stp d5, d6, [sp, #48] 303 stp d7, d8, [sp, #64] 304 stp d9, d10, [sp, #80] 305 stp d11, d12, [sp, #96] 306 stp d13, d14, [sp, #112] 307 stp d15, d16, [sp, #128] 308 stp d17, d18, [sp, #144] 309 stp d19, d20, [sp, #160] 310 stp d21, d22, [sp, #176] 311 stp d23, d24, [sp, #192] 312 stp d25, d26, [sp, #208] 313 stp d27, d28, [sp, #224] 314 stp d29, d30, [sp, #240] 315 str d31, [sp, #256] 316 317 // Save core registers. 318 SAVE_REG x0, 264 319 SAVE_TWO_REGS x1, x2, 272 320 SAVE_TWO_REGS x3, x4, 288 321 SAVE_TWO_REGS x5, x6, 304 322 SAVE_TWO_REGS x7, x8, 320 323 SAVE_TWO_REGS x9, x10, 336 324 SAVE_TWO_REGS x11, x12, 352 325 SAVE_TWO_REGS x13, x14, 368 326 SAVE_TWO_REGS x15, x16, 384 327 SAVE_TWO_REGS x17, x18, 400 328 SAVE_TWO_REGS x19, x20, 416 329 SAVE_TWO_REGS x21, x22, 432 330 SAVE_TWO_REGS x23, x24, 448 331 SAVE_TWO_REGS x25, x26, 464 332 SAVE_TWO_REGS x27, x28, 480 333 334 // art::Runtime** xIP0 = &art::Runtime::instance_ 335 adrp xIP0, :got:_ZN3art7Runtime9instance_E 336 ldr xIP0, [xIP0, #:got_lo12:_ZN3art7Runtime9instance_E] 337 338 ldr xIP0, [xIP0] // art::Runtime* xIP0 = art::Runtime::instance_; 339 340 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveEverything]; 341 ldr xIP0, [xIP0, \runtime_method_offset] 342 343 // Store ArtMethod* Runtime::callee_save_methods_[kSaveEverything]. 344 str xIP0, [sp] 345 // Place sp in Thread::Current()->top_quick_frame. 346 mov xIP0, sp 347 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 348.endm 349 350 /* 351 * Macro that sets up the callee save frame to conform with 352 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 353 */ 354.macro SETUP_SAVE_EVERYTHING_FRAME runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 355 INCREASE_FRAME 512 356 SAVE_TWO_REGS x29, xLR, 496 357 SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR \runtime_method_offset 358.endm 359 360.macro RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 361 // Restore FP registers. 362 // For better performance, load d0 and d31 separately, so that all LDPs are 16-byte aligned. 363 ldr d0, [sp, #8] 364 ldp d1, d2, [sp, #16] 365 ldp d3, d4, [sp, #32] 366 ldp d5, d6, [sp, #48] 367 ldp d7, d8, [sp, #64] 368 ldp d9, d10, [sp, #80] 369 ldp d11, d12, [sp, #96] 370 ldp d13, d14, [sp, #112] 371 ldp d15, d16, [sp, #128] 372 ldp d17, d18, [sp, #144] 373 ldp d19, d20, [sp, #160] 374 ldp d21, d22, [sp, #176] 375 ldp d23, d24, [sp, #192] 376 ldp d25, d26, [sp, #208] 377 ldp d27, d28, [sp, #224] 378 ldp d29, d30, [sp, #240] 379 ldr d31, [sp, #256] 380 381 // Restore core registers, except x0. 382 RESTORE_TWO_REGS x1, x2, 272 383 RESTORE_TWO_REGS x3, x4, 288 384 RESTORE_TWO_REGS x5, x6, 304 385 RESTORE_TWO_REGS x7, x8, 320 386 RESTORE_TWO_REGS x9, x10, 336 387 RESTORE_TWO_REGS x11, x12, 352 388 RESTORE_TWO_REGS x13, x14, 368 389 RESTORE_TWO_REGS x15, x16, 384 390 RESTORE_TWO_REGS x17, x18, 400 391 RESTORE_TWO_REGS x19, x20, 416 392 RESTORE_TWO_REGS x21, x22, 432 393 RESTORE_TWO_REGS x23, x24, 448 394 RESTORE_TWO_REGS x25, x26, 464 395 RESTORE_TWO_REGS x27, x28, 480 396 RESTORE_TWO_REGS x29, xLR, 496 397 398 DECREASE_FRAME 512 399.endm 400 401.macro RESTORE_SAVE_EVERYTHING_FRAME 402 RESTORE_REG x0, 264 403 RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 404.endm 405 406// Macro to refresh the Marking Register (W20). 407// 408// This macro must be called at the end of functions implementing 409// entrypoints that possibly (directly or indirectly) perform a 410// suspend check (before they return). 411.macro REFRESH_MARKING_REGISTER 412#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 413 ldr wMR, [xSELF, #THREAD_IS_GC_MARKING_OFFSET] 414#endif 415.endm 416 417.macro RETURN_IF_RESULT_IS_ZERO 418 cbnz x0, 1f // result non-zero branch over 419 ret // return 4201: 421.endm 422 423.macro RETURN_IF_RESULT_IS_NON_ZERO 424 cbz x0, 1f // result zero branch over 425 ret // return 4261: 427.endm 428 429 /* 430 * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending 431 * exception is Thread::Current()->exception_ when the runtime method frame is ready. 432 */ 433.macro DELIVER_PENDING_EXCEPTION_FRAME_READY 434 mov x0, xSELF 435 436 // Point of no return. 437 bl artDeliverPendingExceptionFromCode // artDeliverPendingExceptionFromCode(Thread*) 438 brk 0 // Unreached 439.endm 440 441 /* 442 * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending 443 * exception is Thread::Current()->exception_. 444 */ 445.macro DELIVER_PENDING_EXCEPTION 446 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME 447 DELIVER_PENDING_EXCEPTION_FRAME_READY 448.endm 449 450.macro RETURN_OR_DELIVER_PENDING_EXCEPTION_REG reg 451 ldr \reg, [xSELF, # THREAD_EXCEPTION_OFFSET] // Get exception field. 452 cbnz \reg, 1f 453 ret 4541: 455 DELIVER_PENDING_EXCEPTION 456.endm 457 458.macro RETURN_OR_DELIVER_PENDING_EXCEPTION 459 RETURN_OR_DELIVER_PENDING_EXCEPTION_REG xIP0 460.endm 461 462// Same as above with x1. This is helpful in stubs that want to avoid clobbering another register. 463.macro RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 464 RETURN_OR_DELIVER_PENDING_EXCEPTION_REG x1 465.endm 466 467.macro RETURN_IF_W0_IS_ZERO_OR_DELIVER 468 cbnz w0, 1f // result non-zero branch over 469 ret // return 4701: 471 DELIVER_PENDING_EXCEPTION 472.endm 473 474.macro NO_ARG_RUNTIME_EXCEPTION c_name, cxx_name 475 .extern \cxx_name 476ENTRY \c_name 477 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context 478 mov x0, xSELF // pass Thread::Current 479 bl \cxx_name // \cxx_name(Thread*) 480 brk 0 481END \c_name 482.endm 483 484.macro NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING c_name, cxx_name 485 .extern \cxx_name 486ENTRY \c_name 487 SETUP_SAVE_EVERYTHING_FRAME // save all registers as basis for long jump context 488 mov x0, xSELF // pass Thread::Current 489 bl \cxx_name // \cxx_name(Thread*) 490 brk 0 491END \c_name 492.endm 493 494.macro ONE_ARG_RUNTIME_EXCEPTION c_name, cxx_name 495 .extern \cxx_name 496ENTRY \c_name 497 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context. 498 mov x1, xSELF // pass Thread::Current. 499 bl \cxx_name // \cxx_name(arg, Thread*). 500 brk 0 501END \c_name 502.endm 503 504.macro TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING c_name, cxx_name 505 .extern \cxx_name 506ENTRY \c_name 507 SETUP_SAVE_EVERYTHING_FRAME // save all registers as basis for long jump context 508 mov x2, xSELF // pass Thread::Current 509 bl \cxx_name // \cxx_name(arg1, arg2, Thread*) 510 brk 0 511END \c_name 512.endm 513 514 /* 515 * Called by managed code, saves callee saves and then calls artThrowException 516 * that will place a mock Method* at the bottom of the stack. Arg1 holds the exception. 517 */ 518ONE_ARG_RUNTIME_EXCEPTION art_quick_deliver_exception, artDeliverExceptionFromCode 519 520 /* 521 * Called by managed code to create and deliver a NullPointerException. 522 */ 523NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_null_pointer_exception, artThrowNullPointerExceptionFromCode 524 525 /* 526 * Call installed by a signal handler to create and deliver a NullPointerException. 527 */ 528 .extern art_quick_throw_null_pointer_exception_from_signal 529ENTRY art_quick_throw_null_pointer_exception_from_signal 530 // The fault handler pushes the gc map address, i.e. "return address", to stack 531 // and passes the fault address in LR. So we need to set up the CFI info accordingly. 532 .cfi_def_cfa_offset __SIZEOF_POINTER__ 533 .cfi_rel_offset lr, 0 534 // Save all registers as basis for long jump context. 535 INCREASE_FRAME (FRAME_SIZE_SAVE_EVERYTHING - __SIZEOF_POINTER__) 536 SAVE_REG x29, (FRAME_SIZE_SAVE_EVERYTHING - 2 * __SIZEOF_POINTER__) // LR already saved. 537 SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR 538 mov x0, lr // pass the fault address stored in LR by the fault handler. 539 mov x1, xSELF // pass Thread::Current. 540 bl artThrowNullPointerExceptionFromSignal // (arg, Thread*). 541 brk 0 542END art_quick_throw_null_pointer_exception_from_signal 543 544 /* 545 * Called by managed code to create and deliver an ArithmeticException. 546 */ 547NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_div_zero, artThrowDivZeroFromCode 548 549 /* 550 * Called by managed code to create and deliver an ArrayIndexOutOfBoundsException. Arg1 holds 551 * index, arg2 holds limit. 552 */ 553TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_array_bounds, artThrowArrayBoundsFromCode 554 555 /* 556 * Called by managed code to create and deliver a StringIndexOutOfBoundsException 557 * as if thrown from a call to String.charAt(). Arg1 holds index, arg2 holds limit. 558 */ 559TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_string_bounds, artThrowStringBoundsFromCode 560 561 /* 562 * Called by managed code to create and deliver a StackOverflowError. 563 */ 564NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFromCode 565 566 /* 567 * All generated callsites for interface invokes and invocation slow paths will load arguments 568 * as usual - except instead of loading arg0/x0 with the target Method*, arg0/x0 will contain 569 * the method_idx. This wrapper will save arg1-arg3, and call the appropriate C helper. 570 * NOTE: "this" is first visible argument of the target, and so can be found in arg1/x1. 571 * 572 * The helper will attempt to locate the target and return a 128-bit result in x0/x1 consisting 573 * of the target Method* in x0 and method->code_ in x1. 574 * 575 * If unsuccessful, the helper will return null/????. There will be a pending exception in the 576 * thread and we branch to another stub to deliver it. 577 * 578 * On success this wrapper will restore arguments and *jump* to the target, leaving the lr 579 * pointing back to the original caller. 580 * 581 * Adapted from ARM32 code. 582 * 583 * Clobbers xIP0. 584 */ 585.macro INVOKE_TRAMPOLINE_BODY cxx_name 586 .extern \cxx_name 587 SETUP_SAVE_REFS_AND_ARGS_FRAME // save callee saves in case allocation triggers GC 588 // Helper signature is always 589 // (method_idx, *this_object, *caller_method, *self, sp) 590 591 mov x2, xSELF // pass Thread::Current 592 mov x3, sp 593 bl \cxx_name // (method_idx, this, Thread*, SP) 594 mov xIP0, x1 // save Method*->code_ 595 RESTORE_SAVE_REFS_AND_ARGS_FRAME 596 REFRESH_MARKING_REGISTER 597 cbz x0, 1f // did we find the target? if not go to exception delivery 598 br xIP0 // tail call to target 5991: 600 DELIVER_PENDING_EXCEPTION 601.endm 602.macro INVOKE_TRAMPOLINE c_name, cxx_name 603ENTRY \c_name 604 INVOKE_TRAMPOLINE_BODY \cxx_name 605END \c_name 606.endm 607 608INVOKE_TRAMPOLINE art_quick_invoke_interface_trampoline_with_access_check, artInvokeInterfaceTrampolineWithAccessCheck 609 610INVOKE_TRAMPOLINE art_quick_invoke_static_trampoline_with_access_check, artInvokeStaticTrampolineWithAccessCheck 611INVOKE_TRAMPOLINE art_quick_invoke_direct_trampoline_with_access_check, artInvokeDirectTrampolineWithAccessCheck 612INVOKE_TRAMPOLINE art_quick_invoke_super_trampoline_with_access_check, artInvokeSuperTrampolineWithAccessCheck 613INVOKE_TRAMPOLINE art_quick_invoke_virtual_trampoline_with_access_check, artInvokeVirtualTrampolineWithAccessCheck 614 615 616.macro INVOKE_STUB_CREATE_FRAME 617 618SAVE_SIZE=15*8 // x4, x5, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, SP, LR, FP saved. 619SAVE_SIZE_AND_METHOD=SAVE_SIZE+8 620 621 622 mov x9, sp // Save stack pointer. 623 .cfi_register sp,x9 624 625 add x10, x2, # SAVE_SIZE_AND_METHOD // calculate size of frame. 626 sub x10, sp, x10 // Calculate SP position - saves + ArtMethod* + args 627 and x10, x10, # ~0xf // Enforce 16 byte stack alignment. 628 mov sp, x10 // Set new SP. 629 630 sub x10, x9, #SAVE_SIZE // Calculate new FP (later). Done here as we must move SP 631 .cfi_def_cfa_register x10 // before this. 632 .cfi_adjust_cfa_offset SAVE_SIZE 633 634 str x28, [x10, #112] 635 .cfi_rel_offset x28, 112 636 637 stp x26, x27, [x10, #96] 638 .cfi_rel_offset x26, 96 639 .cfi_rel_offset x27, 104 640 641 stp x24, x25, [x10, #80] 642 .cfi_rel_offset x24, 80 643 .cfi_rel_offset x25, 88 644 645 stp x22, x23, [x10, #64] 646 .cfi_rel_offset x22, 64 647 .cfi_rel_offset x23, 72 648 649 stp x20, x21, [x10, #48] 650 .cfi_rel_offset x20, 48 651 .cfi_rel_offset x21, 56 652 653 stp x9, x19, [x10, #32] // Save old stack pointer and x19. 654 .cfi_rel_offset sp, 32 655 .cfi_rel_offset x19, 40 656 657 stp x4, x5, [x10, #16] // Save result and shorty addresses. 658 .cfi_rel_offset x4, 16 659 .cfi_rel_offset x5, 24 660 661 stp xFP, xLR, [x10] // Store LR & FP. 662 .cfi_rel_offset x29, 0 663 .cfi_rel_offset x30, 8 664 665 mov xFP, x10 // Use xFP now, as it's callee-saved. 666 .cfi_def_cfa_register x29 667 mov xSELF, x3 // Move thread pointer into SELF register. 668 669 // Copy arguments into stack frame. 670 // Use simple copy routine for now. 671 // 4 bytes per slot. 672 // X1 - source address 673 // W2 - args length 674 // X9 - destination address. 675 // W10 - temporary 676 add x9, sp, #8 // Destination address is bottom of stack + null. 677 678 // Copy parameters into the stack. Use numeric label as this is a macro and Clang's assembler 679 // does not have unique-id variables. 6801: 681 cmp w2, #0 682 beq 2f 683 sub w2, w2, #4 // Need 65536 bytes of range. 684 ldr w10, [x1, x2] 685 str w10, [x9, x2] 686 687 b 1b 688 6892: 690 // Store null into ArtMethod* at bottom of frame. 691 str xzr, [sp] 692.endm 693 694.macro INVOKE_STUB_CALL_AND_RETURN 695 696 REFRESH_MARKING_REGISTER 697 698 // load method-> METHOD_QUICK_CODE_OFFSET 699 ldr x9, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 700 // Branch to method. 701 blr x9 702 703 // Restore return value address and shorty address. 704 ldp x4, x5, [xFP, #16] 705 .cfi_restore x4 706 .cfi_restore x5 707 708 ldr x28, [xFP, #112] 709 .cfi_restore x28 710 711 ldp x26, x27, [xFP, #96] 712 .cfi_restore x26 713 .cfi_restore x27 714 715 ldp x24, x25, [xFP, #80] 716 .cfi_restore x24 717 .cfi_restore x25 718 719 ldp x22, x23, [xFP, #64] 720 .cfi_restore x22 721 .cfi_restore x23 722 723 ldp x20, x21, [xFP, #48] 724 .cfi_restore x20 725 .cfi_restore x21 726 727 // Store result (w0/x0/s0/d0) appropriately, depending on resultType. 728 ldrb w10, [x5] 729 730 // Check the return type and store the correct register into the jvalue in memory. 731 // Use numeric label as this is a macro and Clang's assembler does not have unique-id variables. 732 733 // Don't set anything for a void type. 734 cmp w10, #'V' 735 beq 3f 736 737 // Is it a double? 738 cmp w10, #'D' 739 bne 1f 740 str d0, [x4] 741 b 3f 742 7431: // Is it a float? 744 cmp w10, #'F' 745 bne 2f 746 str s0, [x4] 747 b 3f 748 7492: // Just store x0. Doesn't matter if it is 64 or 32 bits. 750 str x0, [x4] 751 7523: // Finish up. 753 ldp x2, x19, [xFP, #32] // Restore stack pointer and x19. 754 .cfi_restore x19 755 mov sp, x2 756 .cfi_restore sp 757 758 ldp xFP, xLR, [xFP] // Restore old frame pointer and link register. 759 .cfi_restore x29 760 .cfi_restore x30 761 762 ret 763 764.endm 765 766 767/* 768 * extern"C" void art_quick_invoke_stub(ArtMethod *method, x0 769 * uint32_t *args, x1 770 * uint32_t argsize, w2 771 * Thread *self, x3 772 * JValue *result, x4 773 * char *shorty); x5 774 * +----------------------+ 775 * | | 776 * | C/C++ frame | 777 * | LR'' | 778 * | FP'' | <- SP' 779 * +----------------------+ 780 * +----------------------+ 781 * | x28 | <- TODO: Remove callee-saves. 782 * | : | 783 * | x19 | 784 * | SP' | 785 * | X5 | 786 * | X4 | Saved registers 787 * | LR' | 788 * | FP' | <- FP 789 * +----------------------+ 790 * | uint32_t out[n-1] | 791 * | : : | Outs 792 * | uint32_t out[0] | 793 * | ArtMethod* | <- SP value=null 794 * +----------------------+ 795 * 796 * Outgoing registers: 797 * x0 - Method* 798 * x1-x7 - integer parameters. 799 * d0-d7 - Floating point parameters. 800 * xSELF = self 801 * SP = & of ArtMethod* 802 * x1 = "this" pointer. 803 * 804 */ 805ENTRY art_quick_invoke_stub 806 // Spill registers as per AACPS64 calling convention. 807 INVOKE_STUB_CREATE_FRAME 808 809 // Fill registers x/w1 to x/w7 and s/d0 to s/d7 with parameters. 810 // Parse the passed shorty to determine which register to load. 811 // Load addresses for routines that load WXSD registers. 812 adr x11, .LstoreW2 813 adr x12, .LstoreX2 814 adr x13, .LstoreS0 815 adr x14, .LstoreD0 816 817 // Initialize routine offsets to 0 for integers and floats. 818 // x8 for integers, x15 for floating point. 819 mov x8, #0 820 mov x15, #0 821 822 add x10, x5, #1 // Load shorty address, plus one to skip return value. 823 ldr w1, [x9],#4 // Load "this" parameter, and increment arg pointer. 824 825 // Loop to fill registers. 826.LfillRegisters: 827 ldrb w17, [x10], #1 // Load next character in signature, and increment. 828 cbz w17, .LcallFunction // Exit at end of signature. Shorty 0 terminated. 829 830 cmp w17, #'F' // is this a float? 831 bne .LisDouble 832 833 cmp x15, # 8*12 // Skip this load if all registers full. 834 beq .Ladvance4 835 836 add x17, x13, x15 // Calculate subroutine to jump to. 837 br x17 838 839.LisDouble: 840 cmp w17, #'D' // is this a double? 841 bne .LisLong 842 843 cmp x15, # 8*12 // Skip this load if all registers full. 844 beq .Ladvance8 845 846 add x17, x14, x15 // Calculate subroutine to jump to. 847 br x17 848 849.LisLong: 850 cmp w17, #'J' // is this a long? 851 bne .LisOther 852 853 cmp x8, # 6*12 // Skip this load if all registers full. 854 beq .Ladvance8 855 856 add x17, x12, x8 // Calculate subroutine to jump to. 857 br x17 858 859.LisOther: // Everything else takes one vReg. 860 cmp x8, # 6*12 // Skip this load if all registers full. 861 beq .Ladvance4 862 863 add x17, x11, x8 // Calculate subroutine to jump to. 864 br x17 865 866.Ladvance4: 867 add x9, x9, #4 868 b .LfillRegisters 869 870.Ladvance8: 871 add x9, x9, #8 872 b .LfillRegisters 873 874// Macro for loading a parameter into a register. 875// counter - the register with offset into these tables 876// size - the size of the register - 4 or 8 bytes. 877// register - the name of the register to be loaded. 878.macro LOADREG counter size register return 879 ldr \register , [x9], #\size 880 add \counter, \counter, 12 881 b \return 882.endm 883 884// Store ints. 885.LstoreW2: 886 LOADREG x8 4 w2 .LfillRegisters 887 LOADREG x8 4 w3 .LfillRegisters 888 LOADREG x8 4 w4 .LfillRegisters 889 LOADREG x8 4 w5 .LfillRegisters 890 LOADREG x8 4 w6 .LfillRegisters 891 LOADREG x8 4 w7 .LfillRegisters 892 893// Store longs. 894.LstoreX2: 895 LOADREG x8 8 x2 .LfillRegisters 896 LOADREG x8 8 x3 .LfillRegisters 897 LOADREG x8 8 x4 .LfillRegisters 898 LOADREG x8 8 x5 .LfillRegisters 899 LOADREG x8 8 x6 .LfillRegisters 900 LOADREG x8 8 x7 .LfillRegisters 901 902// Store singles. 903.LstoreS0: 904 LOADREG x15 4 s0 .LfillRegisters 905 LOADREG x15 4 s1 .LfillRegisters 906 LOADREG x15 4 s2 .LfillRegisters 907 LOADREG x15 4 s3 .LfillRegisters 908 LOADREG x15 4 s4 .LfillRegisters 909 LOADREG x15 4 s5 .LfillRegisters 910 LOADREG x15 4 s6 .LfillRegisters 911 LOADREG x15 4 s7 .LfillRegisters 912 913// Store doubles. 914.LstoreD0: 915 LOADREG x15 8 d0 .LfillRegisters 916 LOADREG x15 8 d1 .LfillRegisters 917 LOADREG x15 8 d2 .LfillRegisters 918 LOADREG x15 8 d3 .LfillRegisters 919 LOADREG x15 8 d4 .LfillRegisters 920 LOADREG x15 8 d5 .LfillRegisters 921 LOADREG x15 8 d6 .LfillRegisters 922 LOADREG x15 8 d7 .LfillRegisters 923 924 925.LcallFunction: 926 927 INVOKE_STUB_CALL_AND_RETURN 928 929END art_quick_invoke_stub 930 931/* extern"C" 932 * void art_quick_invoke_static_stub(ArtMethod *method, x0 933 * uint32_t *args, x1 934 * uint32_t argsize, w2 935 * Thread *self, x3 936 * JValue *result, x4 937 * char *shorty); x5 938 */ 939ENTRY art_quick_invoke_static_stub 940 // Spill registers as per AACPS64 calling convention. 941 INVOKE_STUB_CREATE_FRAME 942 943 // Fill registers x/w1 to x/w7 and s/d0 to s/d7 with parameters. 944 // Parse the passed shorty to determine which register to load. 945 // Load addresses for routines that load WXSD registers. 946 adr x11, .LstoreW1_2 947 adr x12, .LstoreX1_2 948 adr x13, .LstoreS0_2 949 adr x14, .LstoreD0_2 950 951 // Initialize routine offsets to 0 for integers and floats. 952 // x8 for integers, x15 for floating point. 953 mov x8, #0 954 mov x15, #0 955 956 add x10, x5, #1 // Load shorty address, plus one to skip return value. 957 958 // Loop to fill registers. 959.LfillRegisters2: 960 ldrb w17, [x10], #1 // Load next character in signature, and increment. 961 cbz w17, .LcallFunction2 // Exit at end of signature. Shorty 0 terminated. 962 963 cmp w17, #'F' // is this a float? 964 bne .LisDouble2 965 966 cmp x15, # 8*12 // Skip this load if all registers full. 967 beq .Ladvance4_2 968 969 add x17, x13, x15 // Calculate subroutine to jump to. 970 br x17 971 972.LisDouble2: 973 cmp w17, #'D' // is this a double? 974 bne .LisLong2 975 976 cmp x15, # 8*12 // Skip this load if all registers full. 977 beq .Ladvance8_2 978 979 add x17, x14, x15 // Calculate subroutine to jump to. 980 br x17 981 982.LisLong2: 983 cmp w17, #'J' // is this a long? 984 bne .LisOther2 985 986 cmp x8, # 7*12 // Skip this load if all registers full. 987 beq .Ladvance8_2 988 989 add x17, x12, x8 // Calculate subroutine to jump to. 990 br x17 991 992.LisOther2: // Everything else takes one vReg. 993 cmp x8, # 7*12 // Skip this load if all registers full. 994 beq .Ladvance4_2 995 996 add x17, x11, x8 // Calculate subroutine to jump to. 997 br x17 998 999.Ladvance4_2: 1000 add x9, x9, #4 1001 b .LfillRegisters2 1002 1003.Ladvance8_2: 1004 add x9, x9, #8 1005 b .LfillRegisters2 1006 1007// Store ints. 1008.LstoreW1_2: 1009 LOADREG x8 4 w1 .LfillRegisters2 1010 LOADREG x8 4 w2 .LfillRegisters2 1011 LOADREG x8 4 w3 .LfillRegisters2 1012 LOADREG x8 4 w4 .LfillRegisters2 1013 LOADREG x8 4 w5 .LfillRegisters2 1014 LOADREG x8 4 w6 .LfillRegisters2 1015 LOADREG x8 4 w7 .LfillRegisters2 1016 1017// Store longs. 1018.LstoreX1_2: 1019 LOADREG x8 8 x1 .LfillRegisters2 1020 LOADREG x8 8 x2 .LfillRegisters2 1021 LOADREG x8 8 x3 .LfillRegisters2 1022 LOADREG x8 8 x4 .LfillRegisters2 1023 LOADREG x8 8 x5 .LfillRegisters2 1024 LOADREG x8 8 x6 .LfillRegisters2 1025 LOADREG x8 8 x7 .LfillRegisters2 1026 1027// Store singles. 1028.LstoreS0_2: 1029 LOADREG x15 4 s0 .LfillRegisters2 1030 LOADREG x15 4 s1 .LfillRegisters2 1031 LOADREG x15 4 s2 .LfillRegisters2 1032 LOADREG x15 4 s3 .LfillRegisters2 1033 LOADREG x15 4 s4 .LfillRegisters2 1034 LOADREG x15 4 s5 .LfillRegisters2 1035 LOADREG x15 4 s6 .LfillRegisters2 1036 LOADREG x15 4 s7 .LfillRegisters2 1037 1038// Store doubles. 1039.LstoreD0_2: 1040 LOADREG x15 8 d0 .LfillRegisters2 1041 LOADREG x15 8 d1 .LfillRegisters2 1042 LOADREG x15 8 d2 .LfillRegisters2 1043 LOADREG x15 8 d3 .LfillRegisters2 1044 LOADREG x15 8 d4 .LfillRegisters2 1045 LOADREG x15 8 d5 .LfillRegisters2 1046 LOADREG x15 8 d6 .LfillRegisters2 1047 LOADREG x15 8 d7 .LfillRegisters2 1048 1049 1050.LcallFunction2: 1051 1052 INVOKE_STUB_CALL_AND_RETURN 1053 1054END art_quick_invoke_static_stub 1055 1056 1057 1058/* extern"C" void art_quick_osr_stub(void** stack, x0 1059 * size_t stack_size_in_bytes, x1 1060 * const uin8_t* native_pc, x2 1061 * JValue *result, x3 1062 * char *shorty, x4 1063 * Thread *self) x5 1064 */ 1065ENTRY art_quick_osr_stub 1066SAVE_SIZE=14*8 // x3, x4, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, FP, LR saved. 1067 SAVE_TWO_REGS_INCREASE_FRAME x3, x4, SAVE_SIZE 1068 SAVE_TWO_REGS x19, x20, 16 1069 SAVE_TWO_REGS x21, x22, 32 1070 SAVE_TWO_REGS x23, x24, 48 1071 SAVE_TWO_REGS x25, x26, 64 1072 SAVE_TWO_REGS x27, x28, 80 1073 SAVE_TWO_REGS xFP, xLR, 96 1074 1075 mov xSELF, x5 // Move thread pointer into SELF register. 1076 REFRESH_MARKING_REGISTER 1077 1078 INCREASE_FRAME 16 1079 str xzr, [sp] // Store null for ArtMethod* slot 1080 // Branch to stub. 1081 bl .Losr_entry 1082 .cfi_remember_state 1083 DECREASE_FRAME 16 1084 1085 // Restore saved registers including value address and shorty address. 1086 RESTORE_TWO_REGS x19, x20, 16 1087 RESTORE_TWO_REGS x21, x22, 32 1088 RESTORE_TWO_REGS x23, x24, 48 1089 RESTORE_TWO_REGS x25, x26, 64 1090 RESTORE_TWO_REGS x27, x28, 80 1091 RESTORE_TWO_REGS xFP, xLR, 96 1092 RESTORE_TWO_REGS_DECREASE_FRAME x3, x4, SAVE_SIZE 1093 1094 // Store result (w0/x0/s0/d0) appropriately, depending on resultType. 1095 ldrb w10, [x4] 1096 1097 // Check the return type and store the correct register into the jvalue in memory. 1098 1099 // Don't set anything for a void type. 1100 cmp w10, #'V' 1101 beq .Losr_exit 1102 // Is it a double? 1103 cmp w10, #'D' 1104 beq .Losr_return_double 1105 // Is it a float? 1106 cmp w10, #'F' 1107 beq .Losr_return_float 1108 // Just store x0. Doesn't matter if it is 64 or 32 bits. 1109 str x0, [x3] 1110.Losr_exit: 1111 ret 1112.Losr_return_double: 1113 str d0, [x3] 1114 ret 1115.Losr_return_float: 1116 str s0, [x3] 1117 ret 1118 1119.Losr_entry: 1120 .cfi_restore_state // Reset unwind info so following code unwinds. 1121 .cfi_def_cfa_offset (SAVE_SIZE+16) // workaround for clang bug: 31975598 1122 1123 mov x9, sp // Save stack pointer. 1124 .cfi_def_cfa_register x9 1125 1126 // Update stack pointer for the callee 1127 sub sp, sp, x1 1128 1129 // Update link register slot expected by the callee. 1130 sub w1, w1, #8 1131 str lr, [sp, x1] 1132 1133 // Copy arguments into stack frame. 1134 // Use simple copy routine for now. 1135 // 4 bytes per slot. 1136 // X0 - source address 1137 // W1 - args length 1138 // SP - destination address. 1139 // W10 - temporary 1140.Losr_loop_entry: 1141 cbz w1, .Losr_loop_exit 1142 sub w1, w1, #4 1143 ldr w10, [x0, x1] 1144 str w10, [sp, x1] 1145 b .Losr_loop_entry 1146 1147.Losr_loop_exit: 1148 // Branch to the OSR entry point. 1149 br x2 1150 1151END art_quick_osr_stub 1152 1153 /* 1154 * On entry x0 is uintptr_t* gprs_ and x1 is uint64_t* fprs_ 1155 */ 1156 1157ENTRY art_quick_do_long_jump 1158 // Load FPRs 1159 ldp d0, d1, [x1], #16 1160 ldp d2, d3, [x1], #16 1161 ldp d4, d5, [x1], #16 1162 ldp d6, d7, [x1], #16 1163 ldp d8, d9, [x1], #16 1164 ldp d10, d11, [x1], #16 1165 ldp d12, d13, [x1], #16 1166 ldp d14, d15, [x1], #16 1167 ldp d16, d17, [x1], #16 1168 ldp d18, d19, [x1], #16 1169 ldp d20, d21, [x1], #16 1170 ldp d22, d23, [x1], #16 1171 ldp d24, d25, [x1], #16 1172 ldp d26, d27, [x1], #16 1173 ldp d28, d29, [x1], #16 1174 ldp d30, d31, [x1] 1175 1176 // Load GPRs 1177 // TODO: lots of those are smashed, could optimize. 1178 add x0, x0, #30*8 1179 ldp x30, x1, [x0], #-16 // LR & SP 1180 ldp x28, x29, [x0], #-16 1181 ldp x26, x27, [x0], #-16 1182 ldp x24, x25, [x0], #-16 1183 ldp x22, x23, [x0], #-16 1184 ldp x20, x21, [x0], #-16 1185 ldp x18, x19, [x0], #-16 // X18 & xSELF 1186 ldp x16, x17, [x0], #-16 1187 ldp x14, x15, [x0], #-16 1188 ldp x12, x13, [x0], #-16 1189 ldp x10, x11, [x0], #-16 1190 ldp x8, x9, [x0], #-16 1191 ldp x6, x7, [x0], #-16 1192 ldp x4, x5, [x0], #-16 1193 ldp x2, x3, [x0], #-16 1194 mov sp, x1 1195 1196 REFRESH_MARKING_REGISTER 1197 1198 // Need to load PC, it's at the end (after the space for the unused XZR). Use x1. 1199 ldr x1, [x0, #33*8] 1200 // And the value of x0. 1201 ldr x0, [x0] 1202 1203 br x1 1204END art_quick_do_long_jump 1205 1206 /* 1207 * Entry from managed code that calls artLockObjectFromCode, may block for GC. x0 holds the 1208 * possibly null object to lock. 1209 * 1210 * Derived from arm32 code. 1211 */ 1212 .extern artLockObjectFromCode 1213ENTRY art_quick_lock_object 1214 cbz w0, .Lslow_lock 1215 add x4, x0, #MIRROR_OBJECT_LOCK_WORD_OFFSET // exclusive load/store has no immediate anymore 1216.Lretry_lock: 1217 ldr w2, [xSELF, #THREAD_ID_OFFSET] // TODO: Can the thread ID really change during the loop? 1218 ldaxr w1, [x4] // acquire needed only in most common case 1219 and w3, w1, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED // zero the gc bits 1220 cbnz w3, .Lnot_unlocked // already thin locked 1221 // unlocked case - x1: original lock word that's zero except for the read barrier bits. 1222 orr x2, x1, x2 // x2 holds thread id with count of 0 with preserved read barrier bits 1223 stxr w3, w2, [x4] 1224 cbnz w3, .Llock_stxr_fail // store failed, retry 1225 ret 1226.Lnot_unlocked: // x1: original lock word 1227 lsr w3, w1, LOCK_WORD_STATE_SHIFT 1228 cbnz w3, .Lslow_lock // if either of the top two bits are set, go slow path 1229 eor w2, w1, w2 // lock_word.ThreadId() ^ self->ThreadId() 1230 uxth w2, w2 // zero top 16 bits 1231 cbnz w2, .Lslow_lock // lock word and self thread id's match -> recursive lock 1232 // else contention, go to slow path 1233 and w3, w1, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED // zero the gc bits. 1234 add w2, w3, #LOCK_WORD_THIN_LOCK_COUNT_ONE // increment count in lock word placing in w2 to check overflow 1235 lsr w3, w2, #LOCK_WORD_GC_STATE_SHIFT // if the first gc state bit is set, we overflowed. 1236 cbnz w3, .Lslow_lock // if we overflow the count go slow path 1237 add w2, w1, #LOCK_WORD_THIN_LOCK_COUNT_ONE // increment count for real 1238 stxr w3, w2, [x4] 1239 cbnz w3, .Llock_stxr_fail // store failed, retry 1240 ret 1241.Llock_stxr_fail: 1242 b .Lretry_lock // retry 1243.Lslow_lock: 1244 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case we block 1245 mov x1, xSELF // pass Thread::Current 1246 bl artLockObjectFromCode // (Object* obj, Thread*) 1247 RESTORE_SAVE_REFS_ONLY_FRAME 1248 REFRESH_MARKING_REGISTER 1249 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1250END art_quick_lock_object 1251 1252ENTRY art_quick_lock_object_no_inline 1253 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case we block 1254 mov x1, xSELF // pass Thread::Current 1255 bl artLockObjectFromCode // (Object* obj, Thread*) 1256 RESTORE_SAVE_REFS_ONLY_FRAME 1257 REFRESH_MARKING_REGISTER 1258 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1259END art_quick_lock_object_no_inline 1260 1261 /* 1262 * Entry from managed code that calls artUnlockObjectFromCode and delivers exception on failure. 1263 * x0 holds the possibly null object to lock. 1264 * 1265 * Derived from arm32 code. 1266 */ 1267 .extern artUnlockObjectFromCode 1268ENTRY art_quick_unlock_object 1269 cbz x0, .Lslow_unlock 1270 add x4, x0, #MIRROR_OBJECT_LOCK_WORD_OFFSET // exclusive load/store has no immediate anymore 1271.Lretry_unlock: 1272#ifndef USE_READ_BARRIER 1273 ldr w1, [x4] 1274#else 1275 ldxr w1, [x4] // Need to use atomic instructions for read barrier 1276#endif 1277 lsr w2, w1, LOCK_WORD_STATE_SHIFT 1278 cbnz w2, .Lslow_unlock // if either of the top two bits are set, go slow path 1279 ldr w2, [xSELF, #THREAD_ID_OFFSET] 1280 and w3, w1, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED // zero the gc bits 1281 eor w3, w3, w2 // lock_word.ThreadId() ^ self->ThreadId() 1282 uxth w3, w3 // zero top 16 bits 1283 cbnz w3, .Lslow_unlock // do lock word and self thread id's match? 1284 and w3, w1, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED // zero the gc bits 1285 cmp w3, #LOCK_WORD_THIN_LOCK_COUNT_ONE 1286 bpl .Lrecursive_thin_unlock 1287 // transition to unlocked 1288 and w3, w1, #LOCK_WORD_GC_STATE_MASK_SHIFTED // w3: zero except for the preserved read barrier bits 1289#ifndef USE_READ_BARRIER 1290 stlr w3, [x4] 1291#else 1292 stlxr w2, w3, [x4] // Need to use atomic instructions for read barrier 1293 cbnz w2, .Lunlock_stxr_fail // store failed, retry 1294#endif 1295 ret 1296.Lrecursive_thin_unlock: // w1: original lock word 1297 sub w1, w1, #LOCK_WORD_THIN_LOCK_COUNT_ONE // decrement count 1298#ifndef USE_READ_BARRIER 1299 str w1, [x4] 1300#else 1301 stxr w2, w1, [x4] // Need to use atomic instructions for read barrier 1302 cbnz w2, .Lunlock_stxr_fail // store failed, retry 1303#endif 1304 ret 1305.Lunlock_stxr_fail: 1306 b .Lretry_unlock // retry 1307.Lslow_unlock: 1308 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case exception allocation triggers GC 1309 mov x1, xSELF // pass Thread::Current 1310 bl artUnlockObjectFromCode // (Object* obj, Thread*) 1311 RESTORE_SAVE_REFS_ONLY_FRAME 1312 REFRESH_MARKING_REGISTER 1313 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1314END art_quick_unlock_object 1315 1316ENTRY art_quick_unlock_object_no_inline 1317 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case exception allocation triggers GC 1318 mov x1, xSELF // pass Thread::Current 1319 bl artUnlockObjectFromCode // (Object* obj, Thread*) 1320 RESTORE_SAVE_REFS_ONLY_FRAME 1321 REFRESH_MARKING_REGISTER 1322 RETURN_IF_W0_IS_ZERO_OR_DELIVER 1323END art_quick_unlock_object_no_inline 1324 1325 /* 1326 * Entry from managed code that calls artInstanceOfFromCode and on failure calls 1327 * artThrowClassCastExceptionForObject. 1328 */ 1329 .extern artInstanceOfFromCode 1330 .extern artThrowClassCastExceptionForObject 1331ENTRY art_quick_check_instance_of 1332 // Store arguments and link register 1333 // Stack needs to be 16B aligned on calls. 1334 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 32 1335 SAVE_REG xLR, 24 1336 1337 // Call runtime code 1338 bl artInstanceOfFromCode 1339 1340 // Restore LR. 1341 RESTORE_REG xLR, 24 1342 1343 // Check for exception 1344 cbz x0, .Lthrow_class_cast_exception 1345 1346 // Restore and return 1347 .cfi_remember_state 1348 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 1349 ret 1350 .cfi_restore_state // Reset unwind info so following code unwinds. 1351 .cfi_def_cfa_offset 32 // workaround for clang bug: 31975598 1352 1353.Lthrow_class_cast_exception: 1354 // Restore 1355 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 1356 1357 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context 1358 mov x2, xSELF // pass Thread::Current 1359 bl artThrowClassCastExceptionForObject // (Object*, Class*, Thread*) 1360 brk 0 // We should not return here... 1361END art_quick_check_instance_of 1362 1363// Restore xReg's value from [sp, #offset] if xReg is not the same as xExclude. 1364.macro POP_REG_NE xReg, offset, xExclude 1365 .ifnc \xReg, \xExclude 1366 ldr \xReg, [sp, #\offset] // restore xReg 1367 .cfi_restore \xReg 1368 .endif 1369.endm 1370 1371// Restore xReg1's value from [sp, #offset] if xReg1 is not the same as xExclude. 1372// Restore xReg2's value from [sp, #(offset + 8)] if xReg2 is not the same as xExclude. 1373.macro POP_REGS_NE xReg1, xReg2, offset, xExclude 1374 .ifc \xReg1, \xExclude 1375 ldr \xReg2, [sp, #(\offset + 8)] // restore xReg2 1376 .else 1377 .ifc \xReg2, \xExclude 1378 ldr \xReg1, [sp, #\offset] // restore xReg1 1379 .else 1380 ldp \xReg1, \xReg2, [sp, #\offset] // restore xReg1 and xReg2 1381 .endif 1382 .endif 1383 .cfi_restore \xReg1 1384 .cfi_restore \xReg2 1385.endm 1386 1387 /* 1388 * Macro to insert read barrier, only used in art_quick_aput_obj. 1389 * xDest, wDest and xObj are registers, offset is a defined literal such as 1390 * MIRROR_OBJECT_CLASS_OFFSET. Dest needs both x and w versions of the same register to handle 1391 * name mismatch between instructions. This macro uses the lower 32b of register when possible. 1392 * TODO: When read barrier has a fast path, add heap unpoisoning support for the fast path. 1393 */ 1394.macro READ_BARRIER xDest, wDest, xObj, xTemp, wTemp, offset, number 1395#ifdef USE_READ_BARRIER 1396# ifdef USE_BAKER_READ_BARRIER 1397 ldr \wTemp, [\xObj, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 1398 tbnz \wTemp, #LOCK_WORD_READ_BARRIER_STATE_SHIFT, .Lrb_slowpath\number 1399 // False dependency to avoid needing load/load fence. 1400 add \xObj, \xObj, \xTemp, lsr #32 1401 ldr \wDest, [\xObj, #\offset] // Heap reference = 32b. This also zero-extends to \xDest. 1402 UNPOISON_HEAP_REF \wDest 1403 b .Lrb_exit\number 1404# endif // USE_BAKER_READ_BARRIER 1405.Lrb_slowpath\number: 1406 // Store registers used in art_quick_aput_obj (x0-x4, LR), stack is 16B aligned. 1407 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 48 1408 SAVE_TWO_REGS x2, x3, 16 1409 SAVE_TWO_REGS x4, xLR, 32 1410 1411 // mov x0, \xRef // pass ref in x0 (no-op for now since parameter ref is unused) 1412 .ifnc \xObj, x1 1413 mov x1, \xObj // pass xObj 1414 .endif 1415 mov w2, #\offset // pass offset 1416 bl artReadBarrierSlow // artReadBarrierSlow(ref, xObj, offset) 1417 // No need to unpoison return value in w0, artReadBarrierSlow() would do the unpoisoning. 1418 .ifnc \wDest, w0 1419 mov \wDest, w0 // save return value in wDest 1420 .endif 1421 1422 // Conditionally restore saved registers 1423 POP_REG_NE x0, 0, \xDest 1424 POP_REG_NE x1, 8, \xDest 1425 POP_REG_NE x2, 16, \xDest 1426 POP_REG_NE x3, 24, \xDest 1427 POP_REG_NE x4, 32, \xDest 1428 RESTORE_REG xLR, 40 1429 DECREASE_FRAME 48 1430.Lrb_exit\number: 1431#else 1432 ldr \wDest, [\xObj, #\offset] // Heap reference = 32b. This also zero-extends to \xDest. 1433 UNPOISON_HEAP_REF \wDest 1434#endif // USE_READ_BARRIER 1435.endm 1436 1437#ifdef USE_READ_BARRIER 1438 .extern artReadBarrierSlow 1439#endif 1440ENTRY art_quick_aput_obj 1441 cbz x2, .Ldo_aput_null 1442 READ_BARRIER x3, w3, x0, x3, w3, MIRROR_OBJECT_CLASS_OFFSET, 0 // Heap reference = 32b 1443 // This also zero-extends to x3 1444 READ_BARRIER x3, w3, x3, x4, w4, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, 1 // Heap reference = 32b 1445 // This also zero-extends to x3 1446 READ_BARRIER x4, w4, x2, x4, w4, MIRROR_OBJECT_CLASS_OFFSET, 2 // Heap reference = 32b 1447 // This also zero-extends to x4 1448 cmp w3, w4 // value's type == array's component type - trivial assignability 1449 bne .Lcheck_assignability 1450.Ldo_aput: 1451 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1452 // "Compress" = do nothing 1453 POISON_HEAP_REF w2 1454 str w2, [x3, x1, lsl #2] // Heap reference = 32b 1455 ldr x3, [xSELF, #THREAD_CARD_TABLE_OFFSET] 1456 lsr x0, x0, #CARD_TABLE_CARD_SHIFT 1457 strb w3, [x3, x0] 1458 ret 1459.Ldo_aput_null: 1460 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1461 // "Compress" = do nothing 1462 str w2, [x3, x1, lsl #2] // Heap reference = 32b 1463 ret 1464.Lcheck_assignability: 1465 // Store arguments and link register 1466 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 32 1467 SAVE_TWO_REGS x2, xLR, 16 1468 1469 // Call runtime code 1470 mov x0, x3 // Heap reference, 32b, "uncompress" = do nothing, already zero-extended 1471 mov x1, x4 // Heap reference, 32b, "uncompress" = do nothing, already zero-extended 1472 bl artIsAssignableFromCode 1473 1474 // Check for exception 1475 cbz x0, .Lthrow_array_store_exception 1476 1477 // Restore 1478 .cfi_remember_state 1479 RESTORE_TWO_REGS x2, xLR, 16 1480 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 1481 1482 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1483 // "Compress" = do nothing 1484 POISON_HEAP_REF w2 1485 str w2, [x3, x1, lsl #2] // Heap reference = 32b 1486 ldr x3, [xSELF, #THREAD_CARD_TABLE_OFFSET] 1487 lsr x0, x0, #CARD_TABLE_CARD_SHIFT 1488 strb w3, [x3, x0] 1489 ret 1490 .cfi_restore_state // Reset unwind info so following code unwinds. 1491 .cfi_def_cfa_offset 32 // workaround for clang bug: 31975598 1492.Lthrow_array_store_exception: 1493 RESTORE_TWO_REGS x2, xLR, 16 1494 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 1495 1496 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME 1497 mov x1, x2 // Pass value. 1498 mov x2, xSELF // Pass Thread::Current. 1499 bl artThrowArrayStoreException // (Object*, Object*, Thread*). 1500 brk 0 // Unreached. 1501END art_quick_aput_obj 1502 1503// Macro to facilitate adding new allocation entrypoints. 1504.macro ONE_ARG_DOWNCALL name, entrypoint, return 1505 .extern \entrypoint 1506ENTRY \name 1507 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1508 mov x1, xSELF // pass Thread::Current 1509 bl \entrypoint // (uint32_t type_idx, Method* method, Thread*) 1510 RESTORE_SAVE_REFS_ONLY_FRAME 1511 REFRESH_MARKING_REGISTER 1512 \return 1513END \name 1514.endm 1515 1516// Macro to facilitate adding new allocation entrypoints. 1517.macro TWO_ARG_DOWNCALL name, entrypoint, return 1518 .extern \entrypoint 1519ENTRY \name 1520 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1521 mov x2, xSELF // pass Thread::Current 1522 bl \entrypoint // (uint32_t type_idx, Method* method, Thread*) 1523 RESTORE_SAVE_REFS_ONLY_FRAME 1524 REFRESH_MARKING_REGISTER 1525 \return 1526END \name 1527.endm 1528 1529// Macro to facilitate adding new allocation entrypoints. 1530.macro THREE_ARG_DOWNCALL name, entrypoint, return 1531 .extern \entrypoint 1532ENTRY \name 1533 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1534 mov x3, xSELF // pass Thread::Current 1535 bl \entrypoint 1536 RESTORE_SAVE_REFS_ONLY_FRAME 1537 REFRESH_MARKING_REGISTER 1538 \return 1539END \name 1540.endm 1541 1542// Macro to facilitate adding new allocation entrypoints. 1543.macro FOUR_ARG_DOWNCALL name, entrypoint, return 1544 .extern \entrypoint 1545ENTRY \name 1546 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1547 mov x4, xSELF // pass Thread::Current 1548 bl \entrypoint // 1549 RESTORE_SAVE_REFS_ONLY_FRAME 1550 REFRESH_MARKING_REGISTER 1551 \return 1552END \name 1553.endm 1554 1555// Macros taking opportunity of code similarities for downcalls. 1556.macro ONE_ARG_REF_DOWNCALL name, entrypoint, return 1557 .extern \entrypoint 1558ENTRY \name 1559 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1560 mov x1, xSELF // pass Thread::Current 1561 bl \entrypoint // (uint32_t type_idx, Thread*) 1562 RESTORE_SAVE_REFS_ONLY_FRAME 1563 REFRESH_MARKING_REGISTER 1564 \return 1565END \name 1566.endm 1567 1568.macro TWO_ARG_REF_DOWNCALL name, entrypoint, return 1569 .extern \entrypoint 1570ENTRY \name 1571 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1572 mov x2, xSELF // pass Thread::Current 1573 bl \entrypoint 1574 RESTORE_SAVE_REFS_ONLY_FRAME 1575 REFRESH_MARKING_REGISTER 1576 \return 1577END \name 1578.endm 1579 1580.macro THREE_ARG_REF_DOWNCALL name, entrypoint, return 1581 .extern \entrypoint 1582ENTRY \name 1583 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1584 mov x3, xSELF // pass Thread::Current 1585 bl \entrypoint 1586 RESTORE_SAVE_REFS_ONLY_FRAME 1587 REFRESH_MARKING_REGISTER 1588 \return 1589END \name 1590.endm 1591 1592// Macro for string and type resolution and initialization. 1593.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL name, entrypoint, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 1594 .extern \entrypoint 1595ENTRY \name 1596 SETUP_SAVE_EVERYTHING_FRAME \runtime_method_offset // save everything for stack crawl 1597 mov x1, xSELF // pass Thread::Current 1598 bl \entrypoint // (int32_t index, Thread* self) 1599 cbz w0, 1f // If result is null, deliver the OOME. 1600 .cfi_remember_state 1601 RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 1602 REFRESH_MARKING_REGISTER 1603 ret // return 1604 .cfi_restore_state 1605 .cfi_def_cfa_offset FRAME_SIZE_SAVE_EVERYTHING // workaround for clang bug: 31975598 16061: 1607 DELIVER_PENDING_EXCEPTION_FRAME_READY 1608END \name 1609.endm 1610 1611.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT name, entrypoint 1612 ONE_ARG_SAVE_EVERYTHING_DOWNCALL \name, \entrypoint, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET 1613.endm 1614 1615.macro RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1616 cbz w0, 1f // result zero branch over 1617 ret // return 16181: 1619 DELIVER_PENDING_EXCEPTION 1620.endm 1621 1622 /* 1623 * Entry from managed code that calls artHandleFillArrayDataFromCode and delivers exception on 1624 * failure. 1625 */ 1626TWO_ARG_REF_DOWNCALL art_quick_handle_fill_data, artHandleFillArrayDataFromCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1627 1628 /* 1629 * Entry from managed code when uninitialized static storage, this stub will run the class 1630 * initializer and deliver the exception on error. On success the static storage base is 1631 * returned. 1632 */ 1633ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_static_storage, artInitializeStaticStorageFromCode 1634ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_type, artInitializeTypeFromCode 1635ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode 1636ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode 1637 1638// Note: Functions `art{Get,Set}<Kind>{Static,Instance}FromCompiledCode` are 1639// defined with a macro in runtime/entrypoints/quick/quick_field_entrypoints.cc. 1640 1641ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1642ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1643ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1644ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1645ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1646ONE_ARG_REF_DOWNCALL art_quick_get64_static, artGet64StaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1647ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1648 1649TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1650TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1651TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1652TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1653TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1654TWO_ARG_REF_DOWNCALL art_quick_get64_instance, artGet64InstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1655TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 1656 1657TWO_ARG_REF_DOWNCALL art_quick_set8_static, artSet8StaticFromCompiledCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1658TWO_ARG_REF_DOWNCALL art_quick_set16_static, artSet16StaticFromCompiledCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1659TWO_ARG_REF_DOWNCALL art_quick_set32_static, artSet32StaticFromCompiledCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1660TWO_ARG_REF_DOWNCALL art_quick_set64_static, artSet64StaticFromCompiledCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1661TWO_ARG_REF_DOWNCALL art_quick_set_obj_static, artSetObjStaticFromCompiledCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1662 1663THREE_ARG_REF_DOWNCALL art_quick_set8_instance, artSet8InstanceFromCompiledCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1664THREE_ARG_REF_DOWNCALL art_quick_set16_instance, artSet16InstanceFromCompiledCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1665THREE_ARG_REF_DOWNCALL art_quick_set32_instance, artSet32InstanceFromCompiledCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1666THREE_ARG_REF_DOWNCALL art_quick_set64_instance, artSet64InstanceFromCompiledCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1667THREE_ARG_REF_DOWNCALL art_quick_set_obj_instance, artSetObjInstanceFromCompiledCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER 1668 1669// Generate the allocation entrypoints for each allocator. 1670GENERATE_ALLOC_ENTRYPOINTS_FOR_NON_TLAB_ALLOCATORS 1671// Comment out allocators that have arm64 specific asm. 1672// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB) 1673// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_region_tlab, RegionTLAB) 1674GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_region_tlab, RegionTLAB) 1675// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_region_tlab, RegionTLAB) 1676// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_region_tlab, RegionTLAB) 1677// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_region_tlab, RegionTLAB) 1678// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_region_tlab, RegionTLAB) 1679// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_region_tlab, RegionTLAB) 1680GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_region_tlab, RegionTLAB) 1681GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_region_tlab, RegionTLAB) 1682GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_region_tlab, RegionTLAB) 1683 1684// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB) 1685// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_tlab, TLAB) 1686GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_tlab, TLAB) 1687// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_tlab, TLAB) 1688// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_tlab, TLAB) 1689// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_tlab, TLAB) 1690// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_tlab, TLAB) 1691// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_tlab, TLAB) 1692GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_tlab, TLAB) 1693GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_tlab, TLAB) 1694GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_tlab, TLAB) 1695 1696// If isInitialized=1 then the compiler assumes the object's class has already been initialized. 1697// If isInitialized=0 the compiler can only assume it's been at least resolved. 1698.macro ART_QUICK_ALLOC_OBJECT_ROSALLOC c_name, cxx_name, isInitialized 1699ENTRY \c_name 1700 // Fast path rosalloc allocation. 1701 // x0: type, xSELF(x19): Thread::Current 1702 // x1-x7: free. 1703 ldr x3, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] // Check if the thread local 1704 // allocation stack has room. 1705 // ldp won't work due to large offset. 1706 ldr x4, [xSELF, #THREAD_LOCAL_ALLOC_STACK_END_OFFSET] 1707 cmp x3, x4 1708 bhs .Lslow_path\c_name 1709 ldr w3, [x0, #MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET] // Load the object size (x3) 1710 cmp x3, #ROSALLOC_MAX_THREAD_LOCAL_BRACKET_SIZE // Check if the size is for a thread 1711 // local allocation. Also does the 1712 // finalizable and initialization 1713 // checks. 1714 // When isInitialized == 0, then the class is potentially not yet initialized. 1715 // If the class is not yet initialized, the object size will be very large to force the branch 1716 // below to be taken. 1717 // 1718 // See InitializeClassVisitors in class-inl.h for more details. 1719 bhs .Lslow_path\c_name 1720 // Compute the rosalloc bracket index 1721 // from the size. Since the size is 1722 // already aligned we can combine the 1723 // two shifts together. 1724 add x4, xSELF, x3, lsr #(ROSALLOC_BRACKET_QUANTUM_SIZE_SHIFT - POINTER_SIZE_SHIFT) 1725 // Subtract pointer size since ther 1726 // are no runs for 0 byte allocations 1727 // and the size is already aligned. 1728 ldr x4, [x4, #(THREAD_ROSALLOC_RUNS_OFFSET - __SIZEOF_POINTER__)] 1729 // Load the free list head (x3). This 1730 // will be the return val. 1731 ldr x3, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)] 1732 cbz x3, .Lslow_path\c_name 1733 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber x0 and x1. 1734 ldr x1, [x3, #ROSALLOC_SLOT_NEXT_OFFSET] // Load the next pointer of the head 1735 // and update the list head with the 1736 // next pointer. 1737 str x1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)] 1738 // Store the class pointer in the 1739 // header. This also overwrites the 1740 // next pointer. The offsets are 1741 // asserted to match. 1742 1743#if ROSALLOC_SLOT_NEXT_OFFSET != MIRROR_OBJECT_CLASS_OFFSET 1744#error "Class pointer needs to overwrite next pointer." 1745#endif 1746 POISON_HEAP_REF w0 1747 str w0, [x3, #MIRROR_OBJECT_CLASS_OFFSET] 1748 // Push the new object onto the thread 1749 // local allocation stack and 1750 // increment the thread local 1751 // allocation stack top. 1752 ldr x1, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] 1753 str w3, [x1], #COMPRESSED_REFERENCE_SIZE // (Increment x1 as a side effect.) 1754 str x1, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] 1755 // Decrement the size of the free list 1756 1757 // After this "STR" the object is published to the thread local allocation stack, 1758 // and it will be observable from a runtime internal (eg. Heap::VisitObjects) point of view. 1759 // It is not yet visible to the running (user) compiled code until after the return. 1760 // 1761 // To avoid the memory barrier prior to the "STR", a trick is employed, by differentiating 1762 // the state of the allocation stack slot. It can be a pointer to one of: 1763 // 0) Null entry, because the stack was bumped but the new pointer wasn't written yet. 1764 // (The stack initial state is "null" pointers). 1765 // 1) A partially valid object, with an invalid class pointer to the next free rosalloc slot. 1766 // 2) A fully valid object, with a valid class pointer pointing to a real class. 1767 // Other states are not allowed. 1768 // 1769 // An object that is invalid only temporarily, and will eventually become valid. 1770 // The internal runtime code simply checks if the object is not null or is partial and then 1771 // ignores it. 1772 // 1773 // (Note: The actual check is done by seeing if a non-null object has a class pointer pointing 1774 // to ClassClass, and that the ClassClass's class pointer is self-cyclic. A rosalloc free slot 1775 // "next" pointer is not-cyclic.) 1776 // 1777 // See also b/28790624 for a listing of CLs dealing with this race. 1778 ldr w1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)] 1779 sub x1, x1, #1 1780 // TODO: consider combining this store 1781 // and the list head store above using 1782 // strd. 1783 str w1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)] 1784 1785 mov x0, x3 // Set the return value and return. 1786.if \isInitialized == 0 1787 // This barrier is only necessary when the allocation also requires 1788 // a class initialization check. 1789 // 1790 // If the class is already observably initialized, then new-instance allocations are protected 1791 // from publishing by the compiler which inserts its own StoreStore barrier. 1792 dmb ish 1793 // Use a "dmb ish" fence here because if there are later loads of statics (e.g. class size), 1794 // they should happen-after the implicit initialization check. 1795 // 1796 // TODO: Remove this dmb for class initialization checks (b/36692143) by introducing 1797 // a new observably-initialized class state. 1798.endif 1799 ret 1800.Lslow_path\c_name: 1801 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1802 mov x1, xSELF // pass Thread::Current 1803 bl \cxx_name 1804 RESTORE_SAVE_REFS_ONLY_FRAME 1805 REFRESH_MARKING_REGISTER 1806 RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1807END \c_name 1808.endm 1809 1810ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_resolved_rosalloc, artAllocObjectFromCodeResolvedRosAlloc, /* isInitialized */ 0 1811ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_initialized_rosalloc, artAllocObjectFromCodeInitializedRosAlloc, /* isInitialized */ 1 1812 1813// If isInitialized=1 then the compiler assumes the object's class has already been initialized. 1814// If isInitialized=0 the compiler can only assume it's been at least resolved. 1815.macro ALLOC_OBJECT_TLAB_FAST_PATH_RESOLVED slowPathLabel isInitialized 1816 ldr x4, [xSELF, #THREAD_LOCAL_POS_OFFSET] 1817 ldr x5, [xSELF, #THREAD_LOCAL_END_OFFSET] 1818 ldr w7, [x0, #MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET] // Load the object size (x7). 1819 add x6, x4, x7 // Add object size to tlab pos. 1820 cmp x6, x5 // Check if it fits, overflow works 1821 // since the tlab pos and end are 32 1822 // bit values. 1823 1824 // When isInitialized == 0, then the class is potentially not yet initialized. 1825 // If the class is not yet initialized, the object size will be very large to force the branch 1826 // below to be taken. 1827 // 1828 // See InitializeClassVisitors in class-inl.h for more details. 1829 bhi \slowPathLabel 1830 str x6, [xSELF, #THREAD_LOCAL_POS_OFFSET] // Store new thread_local_pos. 1831 ldr x5, [xSELF, #THREAD_LOCAL_OBJECTS_OFFSET] // Increment thread_local_objects. 1832 add x5, x5, #1 1833 str x5, [xSELF, #THREAD_LOCAL_OBJECTS_OFFSET] 1834 POISON_HEAP_REF w0 1835 str w0, [x4, #MIRROR_OBJECT_CLASS_OFFSET] // Store the class pointer. 1836 // Fence. This is "ish" not "ishst" so 1837 // that the code after this allocation 1838 // site will see the right values in 1839 // the fields of the class. 1840 mov x0, x4 1841.if \isInitialized == 0 1842 // This barrier is only necessary when the allocation also requires 1843 // a class initialization check. 1844 // 1845 // If the class is already observably initialized, then new-instance allocations are protected 1846 // from publishing by the compiler which inserts its own StoreStore barrier. 1847 dmb ish 1848 // Use a "dmb ish" fence here because if there are later loads of statics (e.g. class size), 1849 // they should happen-after the implicit initialization check. 1850 // 1851 // TODO: Remove this dmb for class initialization checks (b/36692143) by introducing 1852 // a new observably-initialized class state. 1853.endif 1854 ret 1855.endm 1856 1857// The common code for art_quick_alloc_object_*region_tlab 1858.macro GENERATE_ALLOC_OBJECT_RESOLVED_TLAB name, entrypoint, isInitialized 1859ENTRY \name 1860 // Fast path region tlab allocation. 1861 // x0: type, xSELF(x19): Thread::Current 1862 // x1-x7: free. 1863 ALLOC_OBJECT_TLAB_FAST_PATH_RESOLVED .Lslow_path\name, \isInitialized 1864.Lslow_path\name: 1865 SETUP_SAVE_REFS_ONLY_FRAME // Save callee saves in case of GC. 1866 mov x1, xSELF // Pass Thread::Current. 1867 bl \entrypoint // (mirror::Class*, Thread*) 1868 RESTORE_SAVE_REFS_ONLY_FRAME 1869 REFRESH_MARKING_REGISTER 1870 RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1871END \name 1872.endm 1873 1874GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_resolved_region_tlab, artAllocObjectFromCodeResolvedRegionTLAB, /* isInitialized */ 0 1875GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_initialized_region_tlab, artAllocObjectFromCodeInitializedRegionTLAB, /* isInitialized */ 1 1876GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_resolved_tlab, artAllocObjectFromCodeResolvedTLAB, /* isInitialized */ 0 1877GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_initialized_tlab, artAllocObjectFromCodeInitializedTLAB, /* isInitialized */ 1 1878 1879.macro ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE slowPathLabel, xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1880 and \xTemp1, \xTemp1, #OBJECT_ALIGNMENT_MASK_TOGGLED64 // Apply alignment mask 1881 // (addr + 7) & ~7. The mask must 1882 // be 64 bits to keep high bits in 1883 // case of overflow. 1884 // Negative sized arrays are handled here since xCount holds a zero extended 32 bit value. 1885 // Negative ints become large 64 bit unsigned ints which will always be larger than max signed 1886 // 32 bit int. Since the max shift for arrays is 3, it can not become a negative 64 bit int. 1887 cmp \xTemp1, #MIN_LARGE_OBJECT_THRESHOLD // Possibly a large object, go slow 1888 bhs \slowPathLabel // path. 1889 1890 ldr \xTemp0, [xSELF, #THREAD_LOCAL_POS_OFFSET] // Check tlab for space, note that 1891 // we use (end - begin) to handle 1892 // negative size arrays. It is 1893 // assumed that a negative size will 1894 // always be greater unsigned than 1895 // region size. 1896 ldr \xTemp2, [xSELF, #THREAD_LOCAL_END_OFFSET] 1897 sub \xTemp2, \xTemp2, \xTemp0 1898 cmp \xTemp1, \xTemp2 1899 1900 // The array class is always initialized here. Unlike new-instance, 1901 // this does not act as a double test. 1902 bhi \slowPathLabel 1903 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber x0 and x1. 1904 // Move old thread_local_pos to x0 1905 // for the return value. 1906 mov x0, \xTemp0 1907 add \xTemp0, \xTemp0, \xTemp1 1908 str \xTemp0, [xSELF, #THREAD_LOCAL_POS_OFFSET] // Store new thread_local_pos. 1909 ldr \xTemp0, [xSELF, #THREAD_LOCAL_OBJECTS_OFFSET] // Increment thread_local_objects. 1910 add \xTemp0, \xTemp0, #1 1911 str \xTemp0, [xSELF, #THREAD_LOCAL_OBJECTS_OFFSET] 1912 POISON_HEAP_REF \wClass 1913 str \wClass, [x0, #MIRROR_OBJECT_CLASS_OFFSET] // Store the class pointer. 1914 str \wCount, [x0, #MIRROR_ARRAY_LENGTH_OFFSET] // Store the array length. 1915 // Fence. 1916// new-array is special. The class is loaded and immediately goes to the Initialized state 1917// before it is published. Therefore the only fence needed is for the publication of the object. 1918// See ClassLinker::CreateArrayClass() for more details. 1919 1920// For publication of the new array, we don't need a 'dmb ishst' here. 1921// The compiler generates 'dmb ishst' for all new-array insts. 1922 ret 1923.endm 1924 1925.macro GENERATE_ALLOC_ARRAY_TLAB name, entrypoint, size_setup 1926ENTRY \name 1927 // Fast path array allocation for region tlab allocation. 1928 // x0: mirror::Class* type 1929 // x1: int32_t component_count 1930 // x2-x7: free. 1931 mov x3, x0 1932 \size_setup x3, w3, x1, w1, x4, w4, x5, w5, x6, w6 1933 ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE .Lslow_path\name, x3, w3, x1, w1, x4, w4, x5, w5, x6, w6 1934.Lslow_path\name: 1935 // x0: mirror::Class* klass 1936 // x1: int32_t component_count 1937 // x2: Thread* self 1938 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1939 mov x2, xSELF // pass Thread::Current 1940 bl \entrypoint 1941 RESTORE_SAVE_REFS_ONLY_FRAME 1942 REFRESH_MARKING_REGISTER 1943 RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1944END \name 1945.endm 1946 1947.macro COMPUTE_ARRAY_SIZE_UNKNOWN xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1948 // Array classes are never finalizable or uninitialized, no need to check. 1949 ldr \wTemp0, [\xClass, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET] // Load component type 1950 UNPOISON_HEAP_REF \wTemp0 1951 ldr \wTemp0, [\xTemp0, #MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET] 1952 lsr \xTemp0, \xTemp0, #PRIMITIVE_TYPE_SIZE_SHIFT_SHIFT // Component size shift is in high 16 1953 // bits. 1954 // xCount is holding a 32 bit value, 1955 // it can not overflow. 1956 lsl \xTemp1, \xCount, \xTemp0 // Calculate data size 1957 // Add array data offset and alignment. 1958 add \xTemp1, \xTemp1, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1959#if MIRROR_LONG_ARRAY_DATA_OFFSET != MIRROR_INT_ARRAY_DATA_OFFSET + 4 1960#error Long array data offset must be 4 greater than int array data offset. 1961#endif 1962 1963 add \xTemp0, \xTemp0, #1 // Add 4 to the length only if the 1964 // component size shift is 3 1965 // (for 64 bit alignment). 1966 and \xTemp0, \xTemp0, #4 1967 add \xTemp1, \xTemp1, \xTemp0 1968.endm 1969 1970.macro COMPUTE_ARRAY_SIZE_8 xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1971 // Add array data offset and alignment. 1972 add \xTemp1, \xCount, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1973.endm 1974 1975.macro COMPUTE_ARRAY_SIZE_16 xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1976 lsl \xTemp1, \xCount, #1 1977 // Add array data offset and alignment. 1978 add \xTemp1, \xTemp1, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1979.endm 1980 1981.macro COMPUTE_ARRAY_SIZE_32 xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1982 lsl \xTemp1, \xCount, #2 1983 // Add array data offset and alignment. 1984 add \xTemp1, \xTemp1, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1985.endm 1986 1987.macro COMPUTE_ARRAY_SIZE_64 xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1988 lsl \xTemp1, \xCount, #3 1989 // Add array data offset and alignment. 1990 add \xTemp1, \xTemp1, #(MIRROR_WIDE_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1991.endm 1992 1993// TODO(ngeoffray): art_quick_alloc_array_resolved_region_tlab is not used for arm64, remove 1994// the entrypoint once all backends have been updated to use the size variants. 1995GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_UNKNOWN 1996GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_8 1997GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_16 1998GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_32 1999GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_64 2000GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_UNKNOWN 2001GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_8 2002GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_16 2003GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_32 2004GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_64 2005 2006 /* 2007 * Called by managed code when the thread has been asked to suspend. 2008 */ 2009 .extern artTestSuspendFromCode 2010ENTRY art_quick_test_suspend 2011 SETUP_SAVE_EVERYTHING_FRAME RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET // save callee saves for stack crawl 2012 mov x0, xSELF 2013 bl artTestSuspendFromCode // (Thread*) 2014 RESTORE_SAVE_EVERYTHING_FRAME 2015 REFRESH_MARKING_REGISTER 2016 ret 2017END art_quick_test_suspend 2018 2019ENTRY art_quick_implicit_suspend 2020 mov x0, xSELF 2021 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves for stack crawl 2022 bl artTestSuspendFromCode // (Thread*) 2023 RESTORE_SAVE_REFS_ONLY_FRAME 2024 REFRESH_MARKING_REGISTER 2025 ret 2026END art_quick_implicit_suspend 2027 2028 /* 2029 * Called by managed code that is attempting to call a method on a proxy class. On entry 2030 * x0 holds the proxy method and x1 holds the receiver; The frame size of the invoked proxy 2031 * method agrees with a ref and args callee save frame. 2032 */ 2033 .extern artQuickProxyInvokeHandler 2034ENTRY art_quick_proxy_invoke_handler 2035 SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_X0 2036 mov x2, xSELF // pass Thread::Current 2037 mov x3, sp // pass SP 2038 bl artQuickProxyInvokeHandler // (Method* proxy method, receiver, Thread*, SP) 2039 ldr x2, [xSELF, THREAD_EXCEPTION_OFFSET] 2040 cbnz x2, .Lexception_in_proxy // success if no exception is pending 2041 RESTORE_SAVE_REFS_AND_ARGS_FRAME // Restore frame 2042 REFRESH_MARKING_REGISTER 2043 fmov d0, x0 // Store result in d0 in case it was float or double 2044 ret // return on success 2045.Lexception_in_proxy: 2046 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2047 DELIVER_PENDING_EXCEPTION 2048END art_quick_proxy_invoke_handler 2049 2050 /* 2051 * Called to resolve an imt conflict. 2052 * x0 is the conflict ArtMethod. 2053 * xIP1 is a hidden argument that holds the target interface method's dex method index. 2054 * 2055 * Note that this stub writes to xIP0, xIP1, x13-x15, and x0. 2056 */ 2057 .extern artLookupResolvedMethod 2058ENTRY art_quick_imt_conflict_trampoline 2059 ldr xIP0, [sp, #0] // Load referrer 2060 // Load the declaring class (without read barrier) and access flags (for obsolete method check). 2061 // The obsolete flag is set with suspended threads, so we do not need an acquire operation here. 2062#if ART_METHOD_ACCESS_FLAGS_OFFSET != ART_METHOD_DECLARING_CLASS_OFFSET + 4 2063#error "Expecting declaring class and access flags to be consecutive for LDP." 2064#endif 2065 ldp wIP0, w15, [xIP0, #ART_METHOD_DECLARING_CLASS_OFFSET] 2066 // If the method is obsolete, just go through the dex cache miss slow path. 2067 tbnz x15, #ACC_OBSOLETE_METHOD_SHIFT, .Limt_conflict_trampoline_dex_cache_miss 2068 ldr wIP0, [xIP0, #MIRROR_CLASS_DEX_CACHE_OFFSET] // Load the DexCache (without read barrier). 2069 UNPOISON_HEAP_REF wIP0 2070 ubfx x15, xIP1, #0, #METHOD_DEX_CACHE_HASH_BITS // Calculate DexCache method slot index. 2071 ldr xIP0, [xIP0, #MIRROR_DEX_CACHE_RESOLVED_METHODS_OFFSET] // Load the resolved methods. 2072 add xIP0, xIP0, x15, lsl #(POINTER_SIZE_SHIFT + 1) // Load DexCache method slot address. 2073 2074 // Relaxed atomic load x14:x15 from the dex cache slot. 2075.Limt_conflict_trampoline_retry_load: 2076 ldxp x14, x15, [xIP0] 2077 stxp w13, x14, x15, [xIP0] 2078 cbnz w13, .Limt_conflict_trampoline_retry_load 2079 2080 cmp x15, xIP1 // Compare method index to see if we had a DexCache method hit. 2081 bne .Limt_conflict_trampoline_dex_cache_miss 2082.Limt_conflict_trampoline_have_interface_method: 2083 ldr xIP1, [x0, #ART_METHOD_JNI_OFFSET_64] // Load ImtConflictTable 2084 ldr x0, [xIP1] // Load first entry in ImtConflictTable. 2085.Limt_table_iterate: 2086 cmp x0, x14 2087 // Branch if found. Benchmarks have shown doing a branch here is better. 2088 beq .Limt_table_found 2089 // If the entry is null, the interface method is not in the ImtConflictTable. 2090 cbz x0, .Lconflict_trampoline 2091 // Iterate over the entries of the ImtConflictTable. 2092 ldr x0, [xIP1, #(2 * __SIZEOF_POINTER__)]! 2093 b .Limt_table_iterate 2094.Limt_table_found: 2095 // We successfully hit an entry in the table. Load the target method 2096 // and jump to it. 2097 ldr x0, [xIP1, #__SIZEOF_POINTER__] 2098 ldr xIP0, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 2099 br xIP0 2100.Lconflict_trampoline: 2101 // Call the runtime stub to populate the ImtConflictTable and jump to the 2102 // resolved method. 2103 mov x0, x14 // Load interface method 2104 INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline 2105.Limt_conflict_trampoline_dex_cache_miss: 2106 // We're not creating a proper runtime method frame here, 2107 // artLookupResolvedMethod() is not allowed to walk the stack. 2108 2109 // Save GPR args and return address, allocate space for FPR args, align stack. 2110 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, (8 * 8 + 8 * 8 + 8 + 8) 2111 SAVE_TWO_REGS x2, x3, 16 2112 SAVE_TWO_REGS x4, x5, 32 2113 SAVE_TWO_REGS x6, x7, 48 2114 SAVE_REG xLR, (8 * 8 + 8 * 8 + 8) 2115 2116 // Save FPR args. 2117 stp d0, d1, [sp, #64] 2118 stp d2, d3, [sp, #80] 2119 stp d4, d5, [sp, #96] 2120 stp d6, d7, [sp, #112] 2121 2122 mov x0, xIP1 // Pass method index. 2123 ldr x1, [sp, #(8 * 8 + 8 * 8 + 8 + 8)] // Pass referrer. 2124 bl artLookupResolvedMethod // (uint32_t method_index, ArtMethod* referrer) 2125 mov x14, x0 // Move the interface method to x14 where the loop above expects it. 2126 2127 // Restore FPR args. 2128 ldp d0, d1, [sp, #64] 2129 ldp d2, d3, [sp, #80] 2130 ldp d4, d5, [sp, #96] 2131 ldp d6, d7, [sp, #112] 2132 2133 // Restore GPR args and return address. 2134 RESTORE_REG xLR, (8 * 8 + 8 * 8 + 8) 2135 RESTORE_TWO_REGS x2, x3, 16 2136 RESTORE_TWO_REGS x4, x5, 32 2137 RESTORE_TWO_REGS x6, x7, 48 2138 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, (8 * 8 + 8 * 8 + 8 + 8) 2139 2140 // If the method wasn't resolved, skip the lookup and go to artInvokeInterfaceTrampoline(). 2141 cbz x14, .Lconflict_trampoline 2142 b .Limt_conflict_trampoline_have_interface_method 2143END art_quick_imt_conflict_trampoline 2144 2145ENTRY art_quick_resolution_trampoline 2146 SETUP_SAVE_REFS_AND_ARGS_FRAME 2147 mov x2, xSELF 2148 mov x3, sp 2149 bl artQuickResolutionTrampoline // (called, receiver, Thread*, SP) 2150 cbz x0, 1f 2151 mov xIP0, x0 // Remember returned code pointer in xIP0. 2152 ldr x0, [sp, #0] // artQuickResolutionTrampoline puts called method in *SP. 2153 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2154 REFRESH_MARKING_REGISTER 2155 br xIP0 21561: 2157 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2158 DELIVER_PENDING_EXCEPTION 2159END art_quick_resolution_trampoline 2160 2161/* 2162 * Generic JNI frame layout: 2163 * 2164 * #-------------------# 2165 * | | 2166 * | caller method... | 2167 * #-------------------# <--- SP on entry 2168 * | Return X30/LR | 2169 * | X29/FP | callee save 2170 * | X28 | callee save 2171 * | X27 | callee save 2172 * | X26 | callee save 2173 * | X25 | callee save 2174 * | X24 | callee save 2175 * | X23 | callee save 2176 * | X22 | callee save 2177 * | X21 | callee save 2178 * | X20 | callee save 2179 * | X19 | callee save 2180 * | X7 | arg7 2181 * | X6 | arg6 2182 * | X5 | arg5 2183 * | X4 | arg4 2184 * | X3 | arg3 2185 * | X2 | arg2 2186 * | X1 | arg1 2187 * | D7 | float arg 8 2188 * | D6 | float arg 7 2189 * | D5 | float arg 6 2190 * | D4 | float arg 5 2191 * | D3 | float arg 4 2192 * | D2 | float arg 3 2193 * | D1 | float arg 2 2194 * | D0 | float arg 1 2195 * | Method* | <- X0 2196 * #-------------------# 2197 * | local ref cookie | // 4B 2198 * | handle scope size | // 4B 2199 * #-------------------# 2200 * | JNI Call Stack | 2201 * #-------------------# <--- SP on native call 2202 * | | 2203 * | Stack for Regs | The trampoline assembly will pop these values 2204 * | | into registers for native call 2205 * #-------------------# 2206 * | Native code ptr | 2207 * #-------------------# 2208 * | Free scratch | 2209 * #-------------------# 2210 * | Ptr to (1) | <--- SP 2211 * #-------------------# 2212 */ 2213 /* 2214 * Called to do a generic JNI down-call 2215 */ 2216ENTRY art_quick_generic_jni_trampoline 2217 SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_X0 2218 2219 // Save SP , so we can have static CFI info. 2220 mov x28, sp 2221 .cfi_def_cfa_register x28 2222 2223 // This looks the same, but is different: this will be updated to point to the bottom 2224 // of the frame when the handle scope is inserted. 2225 mov xFP, sp 2226 2227 mov xIP0, #5120 2228 sub sp, sp, xIP0 2229 2230 // prepare for artQuickGenericJniTrampoline call 2231 // (Thread*, SP) 2232 // x0 x1 <= C calling convention 2233 // xSELF xFP <= where they are 2234 2235 mov x0, xSELF // Thread* 2236 mov x1, xFP 2237 bl artQuickGenericJniTrampoline // (Thread*, sp) 2238 2239 // The C call will have registered the complete save-frame on success. 2240 // The result of the call is: 2241 // x0: pointer to native code, 0 on error. 2242 // x1: pointer to the bottom of the used area of the alloca, can restore stack till there. 2243 2244 // Check for error = 0. 2245 cbz x0, .Lexception_in_native 2246 2247 // Release part of the alloca. 2248 mov sp, x1 2249 2250 // Save the code pointer 2251 mov xIP0, x0 2252 2253 // Load parameters from frame into registers. 2254 // TODO Check with artQuickGenericJniTrampoline. 2255 // Also, check again APPCS64 - the stack arguments are interleaved. 2256 ldp x0, x1, [sp] 2257 ldp x2, x3, [sp, #16] 2258 ldp x4, x5, [sp, #32] 2259 ldp x6, x7, [sp, #48] 2260 2261 ldp d0, d1, [sp, #64] 2262 ldp d2, d3, [sp, #80] 2263 ldp d4, d5, [sp, #96] 2264 ldp d6, d7, [sp, #112] 2265 2266 add sp, sp, #128 2267 2268 blr xIP0 // native call. 2269 2270 // result sign extension is handled in C code 2271 // prepare for artQuickGenericJniEndTrampoline call 2272 // (Thread*, result, result_f) 2273 // x0 x1 x2 <= C calling convention 2274 mov x1, x0 // Result (from saved). 2275 mov x0, xSELF // Thread register. 2276 fmov x2, d0 // d0 will contain floating point result, but needs to go into x2 2277 2278 bl artQuickGenericJniEndTrampoline 2279 2280 // Pending exceptions possible. 2281 ldr x2, [xSELF, THREAD_EXCEPTION_OFFSET] 2282 cbnz x2, .Lexception_in_native 2283 2284 // Tear down the alloca. 2285 mov sp, x28 2286 .cfi_def_cfa_register sp 2287 2288 // Tear down the callee-save frame. 2289 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2290 REFRESH_MARKING_REGISTER 2291 2292 // store into fpr, for when it's a fpr return... 2293 fmov d0, x0 2294 ret 2295 2296.Lexception_in_native: 2297 // Move to x1 then sp to please assembler. 2298 ldr x1, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 2299 add sp, x1, #-1 // Remove the GenericJNI tag. 2300 .cfi_def_cfa_register sp 2301 # This will create a new save-all frame, required by the runtime. 2302 DELIVER_PENDING_EXCEPTION 2303END art_quick_generic_jni_trampoline 2304 2305/* 2306 * Called to bridge from the quick to interpreter ABI. On entry the arguments match those 2307 * of a quick call: 2308 * x0 = method being called/to bridge to. 2309 * x1..x7, d0..d7 = arguments to that method. 2310 */ 2311ENTRY art_quick_to_interpreter_bridge 2312 SETUP_SAVE_REFS_AND_ARGS_FRAME // Set up frame and save arguments. 2313 2314 // x0 will contain mirror::ArtMethod* method. 2315 mov x1, xSELF // How to get Thread::Current() ??? 2316 mov x2, sp 2317 2318 // uint64_t artQuickToInterpreterBridge(mirror::ArtMethod* method, Thread* self, 2319 // mirror::ArtMethod** sp) 2320 bl artQuickToInterpreterBridge 2321 2322 RESTORE_SAVE_REFS_AND_ARGS_FRAME // TODO: no need to restore arguments in this case. 2323 REFRESH_MARKING_REGISTER 2324 2325 fmov d0, x0 2326 2327 RETURN_OR_DELIVER_PENDING_EXCEPTION 2328END art_quick_to_interpreter_bridge 2329 2330/* 2331 * Called to attempt to execute an obsolete method. 2332 */ 2333ONE_ARG_RUNTIME_EXCEPTION art_invoke_obsolete_method_stub, artInvokeObsoleteMethod 2334 2335 2336// 2337// Instrumentation-related stubs 2338// 2339 .extern artInstrumentationMethodEntryFromCode 2340ENTRY art_quick_instrumentation_entry 2341 SETUP_SAVE_REFS_AND_ARGS_FRAME 2342 2343 mov x20, x0 // Preserve method reference in a callee-save. 2344 2345 mov x2, xSELF 2346 mov x3, sp // Pass SP 2347 bl artInstrumentationMethodEntryFromCode // (Method*, Object*, Thread*, SP) 2348 2349 mov xIP0, x0 // x0 = result of call. 2350 mov x0, x20 // Reload method reference. 2351 2352 RESTORE_SAVE_REFS_AND_ARGS_FRAME // Note: will restore xSELF 2353 REFRESH_MARKING_REGISTER 2354 cbz xIP0, 1f // Deliver the pending exception if method is null. 2355 adr xLR, art_quick_instrumentation_exit 2356 br xIP0 // Tail-call method with lr set to art_quick_instrumentation_exit. 2357 23581: 2359 DELIVER_PENDING_EXCEPTION 2360END art_quick_instrumentation_entry 2361 2362 .extern artInstrumentationMethodExitFromCode 2363ENTRY art_quick_instrumentation_exit 2364 mov xLR, #0 // Clobber LR for later checks. 2365 SETUP_SAVE_EVERYTHING_FRAME 2366 2367 add x3, sp, #8 // Pass floating-point result pointer, in kSaveEverything frame. 2368 add x2, sp, #264 // Pass integer result pointer, in kSaveEverything frame. 2369 mov x1, sp // Pass SP. 2370 mov x0, xSELF // Pass Thread. 2371 bl artInstrumentationMethodExitFromCode // (Thread*, SP, gpr_res*, fpr_res*) 2372 2373 cbz x0, .Ldo_deliver_instrumentation_exception 2374 // Handle error 2375 cbnz x1, .Ldeoptimize 2376 // Normal return. 2377 str x0, [sp, #FRAME_SIZE_SAVE_EVERYTHING - 8] 2378 // Set return pc. 2379 RESTORE_SAVE_EVERYTHING_FRAME 2380 REFRESH_MARKING_REGISTER 2381 br lr 2382.Ldo_deliver_instrumentation_exception: 2383 DELIVER_PENDING_EXCEPTION_FRAME_READY 2384.Ldeoptimize: 2385 str x1, [sp, #FRAME_SIZE_SAVE_EVERYTHING - 8] 2386 // Set return pc. 2387 RESTORE_SAVE_EVERYTHING_FRAME 2388 // Jump to art_quick_deoptimize. 2389 b art_quick_deoptimize 2390END art_quick_instrumentation_exit 2391 2392 /* 2393 * Instrumentation has requested that we deoptimize into the interpreter. The deoptimization 2394 * will long jump to the upcall with a special exception of -1. 2395 */ 2396 .extern artDeoptimize 2397ENTRY art_quick_deoptimize 2398 SETUP_SAVE_EVERYTHING_FRAME 2399 mov x0, xSELF // Pass thread. 2400 bl artDeoptimize // (Thread*) 2401 brk 0 2402END art_quick_deoptimize 2403 2404 /* 2405 * Compiled code has requested that we deoptimize into the interpreter. The deoptimization 2406 * will long jump to the upcall with a special exception of -1. 2407 */ 2408 .extern artDeoptimizeFromCompiledCode 2409ENTRY art_quick_deoptimize_from_compiled_code 2410 SETUP_SAVE_EVERYTHING_FRAME 2411 mov x1, xSELF // Pass thread. 2412 bl artDeoptimizeFromCompiledCode // (DeoptimizationKind, Thread*) 2413 brk 0 2414END art_quick_deoptimize_from_compiled_code 2415 2416 2417 /* 2418 * String's indexOf. 2419 * 2420 * TODO: Not very optimized. 2421 * On entry: 2422 * x0: string object (known non-null) 2423 * w1: char to match (known <= 0xFFFF) 2424 * w2: Starting offset in string data 2425 */ 2426ENTRY art_quick_indexof 2427#if (STRING_COMPRESSION_FEATURE) 2428 ldr w4, [x0, #MIRROR_STRING_COUNT_OFFSET] 2429#else 2430 ldr w3, [x0, #MIRROR_STRING_COUNT_OFFSET] 2431#endif 2432 add x0, x0, #MIRROR_STRING_VALUE_OFFSET 2433#if (STRING_COMPRESSION_FEATURE) 2434 /* w4 holds count (with flag) and w3 holds actual length */ 2435 lsr w3, w4, #1 2436#endif 2437 /* Clamp start to [0..count] */ 2438 cmp w2, #0 2439 csel w2, wzr, w2, lt 2440 cmp w2, w3 2441 csel w2, w3, w2, gt 2442 2443 /* Save a copy to compute result */ 2444 mov x5, x0 2445 2446#if (STRING_COMPRESSION_FEATURE) 2447 tbz w4, #0, .Lstring_indexof_compressed 2448#endif 2449 /* Build pointer to start of data to compare and pre-bias */ 2450 add x0, x0, x2, lsl #1 2451 sub x0, x0, #2 2452 /* Compute iteration count */ 2453 sub w2, w3, w2 2454 2455 /* 2456 * At this point we have: 2457 * x0: start of the data to test 2458 * w1: char to compare 2459 * w2: iteration count 2460 * x5: original start of string data 2461 */ 2462 2463 subs w2, w2, #4 2464 b.lt .Lindexof_remainder 2465 2466.Lindexof_loop4: 2467 ldrh w6, [x0, #2]! 2468 ldrh w7, [x0, #2]! 2469 ldrh wIP0, [x0, #2]! 2470 ldrh wIP1, [x0, #2]! 2471 cmp w6, w1 2472 b.eq .Lmatch_0 2473 cmp w7, w1 2474 b.eq .Lmatch_1 2475 cmp wIP0, w1 2476 b.eq .Lmatch_2 2477 cmp wIP1, w1 2478 b.eq .Lmatch_3 2479 subs w2, w2, #4 2480 b.ge .Lindexof_loop4 2481 2482.Lindexof_remainder: 2483 adds w2, w2, #4 2484 b.eq .Lindexof_nomatch 2485 2486.Lindexof_loop1: 2487 ldrh w6, [x0, #2]! 2488 cmp w6, w1 2489 b.eq .Lmatch_3 2490 subs w2, w2, #1 2491 b.ne .Lindexof_loop1 2492 2493.Lindexof_nomatch: 2494 mov x0, #-1 2495 ret 2496 2497.Lmatch_0: 2498 sub x0, x0, #6 2499 sub x0, x0, x5 2500 asr x0, x0, #1 2501 ret 2502.Lmatch_1: 2503 sub x0, x0, #4 2504 sub x0, x0, x5 2505 asr x0, x0, #1 2506 ret 2507.Lmatch_2: 2508 sub x0, x0, #2 2509 sub x0, x0, x5 2510 asr x0, x0, #1 2511 ret 2512.Lmatch_3: 2513 sub x0, x0, x5 2514 asr x0, x0, #1 2515 ret 2516#if (STRING_COMPRESSION_FEATURE) 2517 /* 2518 * Comparing compressed string character-per-character with 2519 * input character 2520 */ 2521.Lstring_indexof_compressed: 2522 add x0, x0, x2 2523 sub x0, x0, #1 2524 sub w2, w3, w2 2525.Lstring_indexof_compressed_loop: 2526 subs w2, w2, #1 2527 b.lt .Lindexof_nomatch 2528 ldrb w6, [x0, #1]! 2529 cmp w6, w1 2530 b.eq .Lstring_indexof_compressed_matched 2531 b .Lstring_indexof_compressed_loop 2532.Lstring_indexof_compressed_matched: 2533 sub x0, x0, x5 2534 ret 2535#endif 2536END art_quick_indexof 2537 2538 /* 2539 * Create a function `name` calling the ReadBarrier::Mark routine, 2540 * getting its argument and returning its result through W register 2541 * `wreg` (corresponding to X register `xreg`), saving and restoring 2542 * all caller-save registers. 2543 * 2544 * If `wreg` is different from `w0`, the generated function follows a 2545 * non-standard runtime calling convention: 2546 * - register `wreg` is used to pass the (sole) argument of this 2547 * function (instead of W0); 2548 * - register `wreg` is used to return the result of this function 2549 * (instead of W0); 2550 * - W0 is treated like a normal (non-argument) caller-save register; 2551 * - everything else is the same as in the standard runtime calling 2552 * convention (e.g. standard callee-save registers are preserved). 2553 */ 2554.macro READ_BARRIER_MARK_REG name, wreg, xreg 2555ENTRY \name 2556 // Reference is null, no work to do at all. 2557 cbz \wreg, .Lret_rb_\name 2558 // Use wIP0 as temp and check the mark bit of the reference. wIP0 is not used by the compiler. 2559 ldr wIP0, [\xreg, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 2560 tbz wIP0, #LOCK_WORD_MARK_BIT_SHIFT, .Lnot_marked_rb_\name 2561.Lret_rb_\name: 2562 ret 2563.Lnot_marked_rb_\name: 2564 // Check if the top two bits are one, if this is the case it is a forwarding address. 2565 tst wIP0, wIP0, lsl #1 2566 bmi .Lret_forwarding_address\name 2567.Lslow_rb_\name: 2568 /* 2569 * Allocate 44 stack slots * 8 = 352 bytes: 2570 * - 20 slots for core registers X0-15, X17-X19, LR 2571 * - 24 slots for floating-point registers D0-D7 and D16-D31 2572 */ 2573 // We must not clobber IP1 since code emitted for HLoadClass and HLoadString 2574 // relies on IP1 being preserved. 2575 // Save all potentially live caller-save core registers. 2576 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 352 2577 SAVE_TWO_REGS x2, x3, 16 2578 SAVE_TWO_REGS x4, x5, 32 2579 SAVE_TWO_REGS x6, x7, 48 2580 SAVE_TWO_REGS x8, x9, 64 2581 SAVE_TWO_REGS x10, x11, 80 2582 SAVE_TWO_REGS x12, x13, 96 2583 SAVE_TWO_REGS x14, x15, 112 2584 SAVE_TWO_REGS x17, x18, 128 // Skip x16, i.e. IP0. 2585 SAVE_TWO_REGS x19, xLR, 144 // Save also return address. 2586 // Save all potentially live caller-save floating-point registers. 2587 stp d0, d1, [sp, #160] 2588 stp d2, d3, [sp, #176] 2589 stp d4, d5, [sp, #192] 2590 stp d6, d7, [sp, #208] 2591 stp d16, d17, [sp, #224] 2592 stp d18, d19, [sp, #240] 2593 stp d20, d21, [sp, #256] 2594 stp d22, d23, [sp, #272] 2595 stp d24, d25, [sp, #288] 2596 stp d26, d27, [sp, #304] 2597 stp d28, d29, [sp, #320] 2598 stp d30, d31, [sp, #336] 2599 2600 .ifnc \wreg, w0 2601 mov w0, \wreg // Pass arg1 - obj from `wreg` 2602 .endif 2603 bl artReadBarrierMark // artReadBarrierMark(obj) 2604 .ifnc \wreg, w0 2605 mov \wreg, w0 // Return result into `wreg` 2606 .endif 2607 2608 // Restore core regs, except `xreg`, as `wreg` is used to return the 2609 // result of this function (simply remove it from the stack instead). 2610 POP_REGS_NE x0, x1, 0, \xreg 2611 POP_REGS_NE x2, x3, 16, \xreg 2612 POP_REGS_NE x4, x5, 32, \xreg 2613 POP_REGS_NE x6, x7, 48, \xreg 2614 POP_REGS_NE x8, x9, 64, \xreg 2615 POP_REGS_NE x10, x11, 80, \xreg 2616 POP_REGS_NE x12, x13, 96, \xreg 2617 POP_REGS_NE x14, x15, 112, \xreg 2618 POP_REGS_NE x17, x18, 128, \xreg 2619 POP_REGS_NE x19, xLR, 144, \xreg // Restore also return address. 2620 // Restore floating-point registers. 2621 ldp d0, d1, [sp, #160] 2622 ldp d2, d3, [sp, #176] 2623 ldp d4, d5, [sp, #192] 2624 ldp d6, d7, [sp, #208] 2625 ldp d16, d17, [sp, #224] 2626 ldp d18, d19, [sp, #240] 2627 ldp d20, d21, [sp, #256] 2628 ldp d22, d23, [sp, #272] 2629 ldp d24, d25, [sp, #288] 2630 ldp d26, d27, [sp, #304] 2631 ldp d28, d29, [sp, #320] 2632 ldp d30, d31, [sp, #336] 2633 // Remove frame and return. 2634 DECREASE_FRAME 352 2635 ret 2636.Lret_forwarding_address\name: 2637 // Shift left by the forwarding address shift. This clears out the state bits since they are 2638 // in the top 2 bits of the lock word. 2639 lsl \wreg, wIP0, #LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT 2640 ret 2641END \name 2642.endm 2643 2644READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg00, w0, x0 2645READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, w1, x1 2646READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, w2, x2 2647READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, w3, x3 2648READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg04, w4, x4 2649READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, w5, x5 2650READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, w6, x6 2651READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, w7, x7 2652READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, w8, x8 2653READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg09, w9, x9 2654READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, w10, x10 2655READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, w11, x11 2656READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg12, w12, x12 2657READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg13, w13, x13 2658READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg14, w14, x14 2659READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg15, w15, x15 2660// READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg16, w16, x16 ip0 is blocked 2661READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg17, w17, x17 2662READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg18, w18, x18 2663READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg19, w19, x19 2664READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg20, w20, x20 2665READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg21, w21, x21 2666READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg22, w22, x22 2667READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg23, w23, x23 2668READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg24, w24, x24 2669READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg25, w25, x25 2670READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg26, w26, x26 2671READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg27, w27, x27 2672READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg28, w28, x28 2673READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg29, w29, x29 2674 2675 2676.macro SELECT_X_OR_W_FOR_MACRO macro_to_use, x, w, xreg 2677 .if \xreg 2678 \macro_to_use \x 2679 .else 2680 \macro_to_use \w 2681 .endif 2682.endm 2683 2684.macro FOR_REGISTERS macro_for_register, macro_for_reserved_register, xreg 2685 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x0, w0, \xreg 2686 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x1, w1, \xreg 2687 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x2, w2, \xreg 2688 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x3, w3, \xreg 2689 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x4, w4, \xreg 2690 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x5, w5, \xreg 2691 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x6, w6, \xreg 2692 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x7, w7, \xreg 2693 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x8, w8, \xreg 2694 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x9, w9, \xreg 2695 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x10, w10, \xreg 2696 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x11, w11, \xreg 2697 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x12, w12, \xreg 2698 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x13, w13, \xreg 2699 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x14, w14, \xreg 2700 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x15, w15, \xreg 2701 \macro_for_reserved_register // IP0 is reserved 2702 \macro_for_reserved_register // IP1 is reserved 2703 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x18, w18, \xreg 2704 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x19, w19, \xreg 2705 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x20, w20, \xreg 2706 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x21, w21, \xreg 2707 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x22, w22, \xreg 2708 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x23, w23, \xreg 2709 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x24, w24, \xreg 2710 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x25, w25, \xreg 2711 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x26, w26, \xreg 2712 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x27, w27, \xreg 2713 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x28, w28, \xreg 2714 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x29, w29, \xreg 2715 \macro_for_reserved_register // lr is reserved 2716 \macro_for_reserved_register // sp is reserved 2717.endm 2718 2719.macro FOR_XREGISTERS macro_for_register, macro_for_reserved_register 2720 FOR_REGISTERS \macro_for_register, \macro_for_reserved_register, /* xreg */ 1 2721.endm 2722 2723.macro FOR_WREGISTERS macro_for_register, macro_for_reserved_register 2724 FOR_REGISTERS \macro_for_register, \macro_for_reserved_register, /* xreg */ 0 2725.endm 2726 2727.macro BRK0_BRK0 2728 brk 0 2729 brk 0 2730.endm 2731 2732#if BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET != BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET 2733#error "Array and field introspection code sharing requires same LDR offset." 2734#endif 2735.macro INTROSPECTION_ARRAY_LOAD index_reg 2736 ldr wIP0, [xIP0, \index_reg, lsl #2] 2737 b art_quick_read_barrier_mark_introspection 2738.endm 2739 2740.macro MOV_WIP0_TO_WREG_AND_BL_LR reg 2741 mov \reg, wIP0 2742 br lr // Do not use RET as we do not enter the entrypoint with "BL". 2743.endm 2744 2745.macro READ_BARRIER_MARK_INTROSPECTION_SLOW_PATH ldr_offset 2746 /* 2747 * Allocate 44 stack slots * 8 = 352 bytes: 2748 * - 19 slots for core registers X0-15, X18-X19, LR 2749 * - 1 slot padding 2750 * - 24 slots for floating-point registers D0-D7 and D16-D31 2751 */ 2752 // Save all potentially live caller-save core registers. 2753 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 352 2754 SAVE_TWO_REGS x2, x3, 16 2755 SAVE_TWO_REGS x4, x5, 32 2756 SAVE_TWO_REGS x6, x7, 48 2757 SAVE_TWO_REGS x8, x9, 64 2758 SAVE_TWO_REGS x10, x11, 80 2759 SAVE_TWO_REGS x12, x13, 96 2760 SAVE_TWO_REGS x14, x15, 112 2761 SAVE_TWO_REGS x18, x19, 128 // Skip x16, x17, i.e. IP0, IP1. 2762 SAVE_REG xLR, 144 // Save return address, skip padding at 152. 2763 // Save all potentially live caller-save floating-point registers. 2764 stp d0, d1, [sp, #160] 2765 stp d2, d3, [sp, #176] 2766 stp d4, d5, [sp, #192] 2767 stp d6, d7, [sp, #208] 2768 stp d16, d17, [sp, #224] 2769 stp d18, d19, [sp, #240] 2770 stp d20, d21, [sp, #256] 2771 stp d22, d23, [sp, #272] 2772 stp d24, d25, [sp, #288] 2773 stp d26, d27, [sp, #304] 2774 stp d28, d29, [sp, #320] 2775 stp d30, d31, [sp, #336] 2776 2777 mov x0, xIP0 2778 bl artReadBarrierMark // artReadBarrierMark(obj) 2779 mov xIP0, x0 2780 2781 // Restore core regs, except x0 and x1 as the return register switch case 2782 // address calculation is smoother with an extra register. 2783 RESTORE_TWO_REGS x2, x3, 16 2784 RESTORE_TWO_REGS x4, x5, 32 2785 RESTORE_TWO_REGS x6, x7, 48 2786 RESTORE_TWO_REGS x8, x9, 64 2787 RESTORE_TWO_REGS x10, x11, 80 2788 RESTORE_TWO_REGS x12, x13, 96 2789 RESTORE_TWO_REGS x14, x15, 112 2790 RESTORE_TWO_REGS x18, x19, 128 // Skip x16, x17, i.e. IP0, IP1. 2791 RESTORE_REG xLR, 144 // Restore return address. 2792 // Restore caller-save floating-point registers. 2793 ldp d0, d1, [sp, #160] 2794 ldp d2, d3, [sp, #176] 2795 ldp d4, d5, [sp, #192] 2796 ldp d6, d7, [sp, #208] 2797 ldp d16, d17, [sp, #224] 2798 ldp d18, d19, [sp, #240] 2799 ldp d20, d21, [sp, #256] 2800 ldp d22, d23, [sp, #272] 2801 ldp d24, d25, [sp, #288] 2802 ldp d26, d27, [sp, #304] 2803 ldp d28, d29, [sp, #320] 2804 ldp d30, d31, [sp, #336] 2805 2806 ldr x0, [lr, #\ldr_offset] // Load the instruction. 2807 adr xIP1, .Lmark_introspection_return_switch 2808 bfi xIP1, x0, #3, #5 // Calculate switch case address. 2809 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 352 2810 br xIP1 2811.endm 2812 2813 /* 2814 * Use introspection to load a reference from the same address as the LDR 2815 * instruction in generated code would load (unless loaded by the thunk, 2816 * see below), call ReadBarrier::Mark() with that reference if needed 2817 * and return it in the same register as the LDR instruction would load. 2818 * 2819 * The entrypoint is called through a thunk that differs across load kinds. 2820 * For field and array loads the LDR instruction in generated code follows 2821 * the branch to the thunk, i.e. the LDR is at [LR, #-4], and the thunk 2822 * knows the holder and performs the gray bit check, returning to the LDR 2823 * instruction if the object is not gray, so this entrypoint no longer 2824 * needs to know anything about the holder. For GC root loads, the LDR 2825 * instruction in generated code precedes the branch to the thunk (i.e. 2826 * the LDR is at [LR, #-8]) and the thunk does not do the gray bit check. 2827 * 2828 * For field accesses and array loads with a constant index the thunk loads 2829 * the reference into IP0 using introspection and calls the main entrypoint, 2830 * art_quick_read_barrier_mark_introspection. With heap poisoning enabled, 2831 * the passed reference is poisoned. 2832 * 2833 * For array accesses with non-constant index, the thunk inserts the bits 2834 * 16-21 of the LDR instruction to the entrypoint address, effectively 2835 * calculating a switch case label based on the index register (bits 16-20) 2836 * and adding an extra offset (bit 21 is set) to differentiate from the 2837 * main entrypoint, then moves the base register to IP0 and jumps to the 2838 * switch case. Therefore we need to align the main entrypoint to 512 bytes, 2839 * accounting for a 256-byte offset followed by 32 array entrypoints 2840 * starting at art_quick_read_barrier_mark_introspection_arrays, each 2841 * containing an LDR (register) and a branch to the main entrypoint. 2842 * 2843 * For GC root accesses we cannot use the main entrypoint because of the 2844 * different offset where the LDR instruction in generated code is located. 2845 * (And even with heap poisoning enabled, GC roots are not poisoned.) 2846 * To re-use the same entrypoint pointer in generated code, we make sure 2847 * that the gc root entrypoint (a copy of the entrypoint with a different 2848 * offset for introspection loads) is located at a known offset (768 bytes, 2849 * or BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET) from the main 2850 * entrypoint and the GC root thunk adjusts the entrypoint pointer, moves 2851 * the root register to IP0 and jumps to the customized entrypoint, 2852 * art_quick_read_barrier_mark_introspection_gc_roots. The thunk also 2853 * performs all the fast-path checks, so we need just the slow path. 2854 * 2855 * The code structure is 2856 * art_quick_read_barrier_mark_introspection: 2857 * Up to 256 bytes for the main entrypoint code. 2858 * Padding to 256 bytes if needed. 2859 * art_quick_read_barrier_mark_introspection_arrays: 2860 * Exactly 256 bytes for array load switch cases (32x2 instructions). 2861 * .Lmark_introspection_return_switch: 2862 * Exactly 256 bytes for return switch cases (32x2 instructions). 2863 * art_quick_read_barrier_mark_introspection_gc_roots: 2864 * GC root entrypoint code. 2865 */ 2866 .balign 512 2867ENTRY art_quick_read_barrier_mark_introspection 2868 // At this point, IP0 contains the reference, IP1 can be freely used. 2869 // For heap poisoning, the reference is poisoned, so unpoison it first. 2870 UNPOISON_HEAP_REF wIP0 2871 // If reference is null, just return it in the right register. 2872 cbz wIP0, .Lmark_introspection_return 2873 // Use wIP1 as temp and check the mark bit of the reference. 2874 ldr wIP1, [xIP0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 2875 tbz wIP1, #LOCK_WORD_MARK_BIT_SHIFT, .Lmark_introspection_unmarked 2876.Lmark_introspection_return: 2877 // Without an extra register for the return switch case address calculation, 2878 // we exploit the high word of the xIP0 to temporarily store the ref_reg*8, 2879 // so the return switch below must move wIP0 instead of xIP0 to the register. 2880 ldr wIP1, [lr, #BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET] // Load the instruction. 2881 bfi xIP0, xIP1, #(32 + 3), #5 // Extract ref_reg*8 to high word in xIP0. 2882 adr xIP1, .Lmark_introspection_return_switch 2883 bfxil xIP1, xIP0, #32, #8 // Calculate return switch case address. 2884 br xIP1 2885.Lmark_introspection_unmarked: 2886 // Check if the top two bits are one, if this is the case it is a forwarding address. 2887 tst wIP1, wIP1, lsl #1 2888 bmi .Lmark_introspection_forwarding_address 2889 READ_BARRIER_MARK_INTROSPECTION_SLOW_PATH BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET 2890 2891.Lmark_introspection_forwarding_address: 2892 // Shift left by the forwarding address shift. This clears out the state bits since they are 2893 // in the top 2 bits of the lock word. 2894 lsl wIP0, wIP1, #LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT 2895 b .Lmark_introspection_return 2896 2897 // We're very close to the alloted 256B for the entrypoint code before the 2898 // array switch cases. Should we go a little bit over the limit, we can 2899 // move some code after the array switch cases and return switch cases. 2900 .balign 256 2901 .hidden art_quick_read_barrier_mark_introspection_arrays 2902 .global art_quick_read_barrier_mark_introspection_arrays 2903art_quick_read_barrier_mark_introspection_arrays: 2904 FOR_XREGISTERS INTROSPECTION_ARRAY_LOAD, BRK0_BRK0 2905.Lmark_introspection_return_switch: 2906 FOR_WREGISTERS MOV_WIP0_TO_WREG_AND_BL_LR, BRK0_BRK0 2907 .hidden art_quick_read_barrier_mark_introspection_gc_roots 2908 .global art_quick_read_barrier_mark_introspection_gc_roots 2909art_quick_read_barrier_mark_introspection_gc_roots: 2910 READ_BARRIER_MARK_INTROSPECTION_SLOW_PATH BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_OFFSET 2911END art_quick_read_barrier_mark_introspection 2912 2913.extern artInvokePolymorphic 2914ENTRY art_quick_invoke_polymorphic 2915 SETUP_SAVE_REFS_AND_ARGS_FRAME // Save callee saves in case allocation triggers GC. 2916 mov x2, xSELF 2917 mov x3, sp 2918 INCREASE_FRAME 16 // Reserve space for JValue result. 2919 str xzr, [sp, #0] // Initialize result to zero. 2920 mov x0, sp // Set r0 to point to result. 2921 bl artInvokePolymorphic // artInvokePolymorphic(result, receiver, thread, save_area) 2922 uxtb w0, w0 // Result is the return type descriptor as a char. 2923 sub w0, w0, 'A' // Convert to zero based index. 2924 cmp w0, 'Z' - 'A' 2925 bhi .Lcleanup_and_return // Clean-up if out-of-bounds. 2926 adrp x1, .Lhandler_table // Compute address of handler table. 2927 add x1, x1, :lo12:.Lhandler_table 2928 ldrb w0, [x1, w0, uxtw] // Lookup handler offset in handler table. 2929 adr x1, .Lstart_of_handlers 2930 add x0, x1, w0, sxtb #2 // Convert relative offset to absolute address. 2931 br x0 // Branch to handler. 2932 2933.Lstart_of_handlers: 2934.Lstore_boolean_result: 2935 ldrb w0, [sp] 2936 b .Lcleanup_and_return 2937.Lstore_char_result: 2938 ldrh w0, [sp] 2939 b .Lcleanup_and_return 2940.Lstore_float_result: 2941 ldr s0, [sp] 2942 str s0, [sp, #32] 2943 b .Lcleanup_and_return 2944.Lstore_double_result: 2945 ldr d0, [sp] 2946 str d0, [sp, #32] 2947 b .Lcleanup_and_return 2948.Lstore_long_result: 2949 ldr x0, [sp] 2950 // Fall-through 2951.Lcleanup_and_return: 2952 DECREASE_FRAME 16 2953 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2954 REFRESH_MARKING_REGISTER 2955 RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 2956 2957 .section .rodata // Place handler table in read-only section away from text. 2958 .align 2 2959.macro HANDLER_TABLE_OFFSET handler_label 2960 .byte (\handler_label - .Lstart_of_handlers) / 4 2961.endm 2962.Lhandler_table: 2963 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // A 2964 HANDLER_TABLE_OFFSET(.Lstore_long_result) // B (byte) 2965 HANDLER_TABLE_OFFSET(.Lstore_char_result) // C (char) 2966 HANDLER_TABLE_OFFSET(.Lstore_double_result) // D (double) 2967 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // E 2968 HANDLER_TABLE_OFFSET(.Lstore_float_result) // F (float) 2969 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // G 2970 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // H 2971 HANDLER_TABLE_OFFSET(.Lstore_long_result) // I (int) 2972 HANDLER_TABLE_OFFSET(.Lstore_long_result) // J (long) 2973 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // K 2974 HANDLER_TABLE_OFFSET(.Lstore_long_result) // L (object - references are compressed and only 32-bits) 2975 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // M 2976 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // N 2977 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // O 2978 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // P 2979 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // Q 2980 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // R 2981 HANDLER_TABLE_OFFSET(.Lstore_long_result) // S (short) 2982 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // T 2983 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // U 2984 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // V (void) 2985 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // W 2986 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // X 2987 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // Y 2988 HANDLER_TABLE_OFFSET(.Lstore_boolean_result) // Z (boolean) 2989 .text 2990 2991END art_quick_invoke_polymorphic 2992 2993// Wrap ExecuteSwitchImpl in assembly method which specifies DEX PC for unwinding. 2994// Argument 0: x0: The context pointer for ExecuteSwitchImpl. 2995// Argument 1: x1: Pointer to the templated ExecuteSwitchImpl to call. 2996// Argument 2: x2: The value of DEX PC (memory address of the methods bytecode). 2997ENTRY ExecuteSwitchImplAsm 2998 SAVE_TWO_REGS_INCREASE_FRAME x19, xLR, 16 2999 mov x19, x2 // x19 = DEX PC 3000 CFI_DEFINE_DEX_PC_WITH_OFFSET(0 /* x0 */, 19 /* x19 */, 0) 3001 blr x1 // Call the wrapped method. 3002 RESTORE_TWO_REGS_DECREASE_FRAME x19, xLR, 16 3003 ret 3004END ExecuteSwitchImplAsm 3005