1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#ifndef ART_RUNTIME_ARCH_ARM64_ASM_SUPPORT_ARM64_S_ 18#define ART_RUNTIME_ARCH_ARM64_ASM_SUPPORT_ARM64_S_ 19 20#include "asm_support_arm64.h" 21#include "interpreter/cfi_asm_support.h" 22 23// Define special registers. 24 25// Register holding Thread::Current(). 26#define xSELF x19 27// Frame Pointer 28#define xFP x29 29// Link Register 30#define xLR x30 31// Define the intraprocedural linkage temporary registers. 32#define xIP0 x16 33#define wIP0 w16 34#define xIP1 x17 35#define wIP1 w17 36 37#ifdef RESERVE_MARKING_REGISTER 38// Marking Register, holding Thread::Current()->GetIsGcMarking(). 39#define wMR w20 40#endif 41 42// Implicit suspend check register. 43#define xSUSPEND x21 44 45.macro CFI_EXPRESSION_BREG n, b, offset 46 .if (-0x40 <= (\offset)) && ((\offset) < 0x40) 47 CFI_EXPRESSION_BREG_1(\n, \b, \offset) 48 .elseif (-0x2000 <= (\offset)) && ((\offset) < 0x2000) 49 CFI_EXPRESSION_BREG_2(\n, \b, \offset) 50 .else 51 .error "Unsupported offset" 52 .endif 53.endm 54 55.macro CFI_DEF_CFA_BREG_PLUS_UCONST reg, offset, size 56 .if ((\size) < 0) 57 .error "Size should be positive" 58 .endif 59 .if (((\offset) < -0x40) || ((\offset) >= 0x40)) 60 .error "Unsupported offset" 61 .endif 62 .if ((\size) < 0x80) 63 CFI_DEF_CFA_BREG_PLUS_UCONST_1_1(\reg, \offset, \size) 64 .elseif ((\size) < 0x4000) 65 CFI_DEF_CFA_BREG_PLUS_UCONST_1_2(\reg, \offset, \size) 66 .else 67 .error "Unsupported size" 68 .endif 69.endm 70 71.macro CFI_REMEMBER_STATE 72 .cfi_remember_state 73.endm 74 75// The spec is not clear whether the CFA is part of the saved state and tools 76// differ in the behaviour, so explicitly set the CFA to avoid any ambiguity. 77// The restored CFA state should match the CFA state during CFI_REMEMBER_STATE. 78.macro CFI_RESTORE_STATE_AND_DEF_CFA reg, offset 79 .cfi_restore_state 80 .cfi_def_cfa \reg, \offset 81.endm 82 83.macro ENTRY_ALIGNED name, alignment 84 .type \name, #function 85 .hidden \name // Hide this as a global symbol, so we do not incur plt calls. 86 .global \name 87 .balign \alignment 88\name: 89 .cfi_startproc 90.endm 91 92.macro ENTRY name 93 ENTRY_ALIGNED \name, 16 94.endm 95 96.macro END name 97 .cfi_endproc 98 .size \name, .-\name 99.endm 100 101.macro UNIMPLEMENTED name 102 ENTRY \name 103 brk 0 104 END \name 105.endm 106 107// Macro to poison (negate) the reference for heap poisoning. 108.macro POISON_HEAP_REF rRef 109#ifdef USE_HEAP_POISONING 110 neg \rRef, \rRef 111#endif // USE_HEAP_POISONING 112.endm 113 114// Macro to unpoison (negate) the reference for heap poisoning. 115.macro UNPOISON_HEAP_REF rRef 116#ifdef USE_HEAP_POISONING 117 neg \rRef, \rRef 118#endif // USE_HEAP_POISONING 119.endm 120 121.macro INCREASE_FRAME frame_adjustment 122 sub sp, sp, #(\frame_adjustment) 123 .cfi_adjust_cfa_offset (\frame_adjustment) 124.endm 125 126.macro DECREASE_FRAME frame_adjustment 127 add sp, sp, #(\frame_adjustment) 128 .cfi_adjust_cfa_offset -(\frame_adjustment) 129.endm 130 131.macro SAVE_REG reg, offset 132 str \reg, [sp, #(\offset)] 133 .cfi_rel_offset \reg, (\offset) 134.endm 135 136.macro RESTORE_REG_BASE base, reg, offset 137 ldr \reg, [\base, #(\offset)] 138 .cfi_restore \reg 139.endm 140 141.macro RESTORE_REG reg, offset 142 RESTORE_REG_BASE sp, \reg, \offset 143.endm 144 145.macro SAVE_TWO_REGS_BASE base, reg1, reg2, offset 146 stp \reg1, \reg2, [\base, #(\offset)] 147 .cfi_rel_offset \reg1, (\offset) 148 .cfi_rel_offset \reg2, (\offset) + 8 149.endm 150 151.macro SAVE_TWO_REGS reg1, reg2, offset 152 SAVE_TWO_REGS_BASE sp, \reg1, \reg2, \offset 153.endm 154 155.macro RESTORE_TWO_REGS_BASE base, reg1, reg2, offset 156 ldp \reg1, \reg2, [\base, #(\offset)] 157 .cfi_restore \reg1 158 .cfi_restore \reg2 159.endm 160 161.macro RESTORE_TWO_REGS reg1, reg2, offset 162 RESTORE_TWO_REGS_BASE sp, \reg1, \reg2, \offset 163.endm 164 165.macro LOAD_RUNTIME_INSTANCE reg 166#if __has_feature(hwaddress_sanitizer) 167 adrp \reg, :pg_hi21_nc:_ZN3art7Runtime9instance_E 168#else 169 adrp \reg, _ZN3art7Runtime9instance_E 170#endif 171 ldr \reg, [\reg, #:lo12:_ZN3art7Runtime9instance_E] 172.endm 173 174// Macro to refresh the Marking Register (W20). 175// 176// This macro must be called at the end of functions implementing 177// entrypoints that possibly (directly or indirectly) perform a 178// suspend check (before they return). 179.macro REFRESH_MARKING_REGISTER 180#ifdef RESERVE_MARKING_REGISTER 181 ldr wMR, [xSELF, #THREAD_IS_GC_MARKING_OFFSET] 182#endif 183.endm 184 185// Macro to refresh the suspend check register. 186// 187// We do not refresh `xSUSPEND` after every transition to Runnable, so there is 188// a chance that an implicit suspend check loads null to xSUSPEND but before 189// causing a SIGSEGV at the next implicit suspend check we make a runtime call 190// that performs the suspend check explicitly. This can cause a spurious fault 191// without a pending suspend check request but it should be rare and the fault 192// overhead was already expected when we triggered the suspend check, we just 193// pay the price later than expected. 194.macro REFRESH_SUSPEND_CHECK_REGISTER 195 ldr xSUSPEND, [xSELF, #THREAD_SUSPEND_TRIGGER_OFFSET] 196.endm 197 198 /* 199 * Macro that sets up the callee save frame to conform with 200 * Runtime::CreateCalleeSaveMethod(kSaveRefsOnly). 201 */ 202.macro SETUP_SAVE_REFS_ONLY_FRAME 203 // art::Runtime* xIP0 = art::Runtime::instance_; 204 // Our registers aren't intermixed - just spill in order. 205 LOAD_RUNTIME_INSTANCE xIP0 206 207 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveRefOnly]; 208 ldr xIP0, [xIP0, RUNTIME_SAVE_REFS_ONLY_METHOD_OFFSET] 209 210 INCREASE_FRAME 96 211 212 // Ugly compile-time check, but we only have the preprocessor. 213#if (FRAME_SIZE_SAVE_REFS_ONLY != 96) 214#error "FRAME_SIZE_SAVE_REFS_ONLY(ARM64) size not as expected." 215#endif 216 217 // GP callee-saves. 218 // x20 paired with ArtMethod* - see below. 219 SAVE_TWO_REGS x21, x22, 16 220 SAVE_TWO_REGS x23, x24, 32 221 SAVE_TWO_REGS x25, x26, 48 222 SAVE_TWO_REGS x27, x28, 64 223 SAVE_TWO_REGS x29, xLR, 80 224 225 // Store ArtMethod* Runtime::callee_save_methods_[kSaveRefsOnly]. 226 // Note: We could avoid saving X20 in the case of Baker read 227 // barriers, as it is overwritten by REFRESH_MARKING_REGISTER 228 // later; but it's not worth handling this special case. 229 stp xIP0, x20, [sp] 230 .cfi_rel_offset x20, 8 231 232 // Place sp in Thread::Current()->top_quick_frame. 233 mov xIP0, sp 234 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 235.endm 236 237// TODO: Probably no need to restore registers preserved by aapcs64. 238.macro RESTORE_SAVE_REFS_ONLY_FRAME 239 // Callee-saves. 240 // Note: Likewise, we could avoid restoring X20 in the case of Baker 241 // read barriers, as it is overwritten by REFRESH_MARKING_REGISTER 242 // later; but it's not worth handling this special case. 243 RESTORE_REG x20, 8 244 RESTORE_TWO_REGS x21, x22, 16 245 RESTORE_TWO_REGS x23, x24, 32 246 RESTORE_TWO_REGS x25, x26, 48 247 RESTORE_TWO_REGS x27, x28, 64 248 RESTORE_TWO_REGS x29, xLR, 80 249 250 DECREASE_FRAME 96 251.endm 252 253.macro SETUP_SAVE_REFS_AND_ARGS_FRAME_INTERNAL base 254 // Ugly compile-time check, but we only have the preprocessor. 255#if (FRAME_SIZE_SAVE_REFS_AND_ARGS != 224) 256#error "FRAME_SIZE_SAVE_REFS_AND_ARGS(ARM64) size not as expected." 257#endif 258 259 // Stack alignment filler [\base, #8]. 260 // FP args. 261 stp d0, d1, [\base, #16] 262 stp d2, d3, [\base, #32] 263 stp d4, d5, [\base, #48] 264 stp d6, d7, [\base, #64] 265 266 // Core args. 267 stp x1, x2, [\base, #80] 268 stp x3, x4, [\base, #96] 269 stp x5, x6, [\base, #112] 270 271 // x7, Callee-saves. 272 // Note: We could avoid saving X20 in the case of Baker read 273 // barriers, as it is overwritten by REFRESH_MARKING_REGISTER 274 // later; but it's not worth handling this special case. 275 stp x7, x20, [\base, #128] 276 .cfi_rel_offset x20, 136 277 SAVE_TWO_REGS_BASE \base, x21, x22, 144 278 SAVE_TWO_REGS_BASE \base, x23, x24, 160 279 SAVE_TWO_REGS_BASE \base, x25, x26, 176 280 SAVE_TWO_REGS_BASE \base, x27, x28, 192 281 282 // x29(callee-save) and LR. 283 SAVE_TWO_REGS_BASE \base, x29, xLR, 208 284.endm 285 286// TODO: Probably no need to restore registers preserved by aapcs64. (That would require 287// auditing all users to make sure they restore aapcs64 callee-save registers they clobber.) 288.macro RESTORE_SAVE_REFS_AND_ARGS_FRAME_INTERNAL base 289 // FP args. 290 ldp d0, d1, [\base, #16] 291 ldp d2, d3, [\base, #32] 292 ldp d4, d5, [\base, #48] 293 ldp d6, d7, [\base, #64] 294 295 // Core args. 296 ldp x1, x2, [\base, #80] 297 ldp x3, x4, [\base, #96] 298 ldp x5, x6, [\base, #112] 299 300 // x7, callee-saves and LR. 301 // Note: Likewise, we could avoid restoring X20 in the case of Baker 302 // read barriers, as it is overwritten by REFRESH_MARKING_REGISTER 303 // later; but it's not worth handling this special case. 304 ldp x7, x20, [\base, #128] 305 .cfi_restore x20 306 RESTORE_TWO_REGS_BASE \base, x21, x22, 144 307 RESTORE_TWO_REGS_BASE \base, x23, x24, 160 308 RESTORE_TWO_REGS_BASE \base, x25, x26, 176 309 RESTORE_TWO_REGS_BASE \base, x27, x28, 192 310 RESTORE_TWO_REGS_BASE \base, x29, xLR, 208 311.endm 312 313.macro RESTORE_SAVE_REFS_AND_ARGS_FRAME 314 RESTORE_SAVE_REFS_AND_ARGS_FRAME_INTERNAL sp 315 DECREASE_FRAME FRAME_SIZE_SAVE_REFS_AND_ARGS 316.endm 317 318.macro SAVE_ALL_CALLEE_SAVES offset 319 // FP callee-saves. 320 stp d8, d9, [sp, #(0 + \offset)] 321 stp d10, d11, [sp, #(16 + \offset)] 322 stp d12, d13, [sp, #(32 + \offset)] 323 stp d14, d15, [sp, #(48 + \offset)] 324 325 // GP callee-saves 326 SAVE_TWO_REGS x19, x20, (64 + \offset) 327 SAVE_TWO_REGS x21, x22, (80 + \offset) 328 SAVE_TWO_REGS x23, x24, (96 + \offset) 329 SAVE_TWO_REGS x25, x26, (112 + \offset) 330 SAVE_TWO_REGS x27, x28, (128 + \offset) 331 SAVE_TWO_REGS x29, xLR, (144 + \offset) 332.endm 333 334 /* 335 * Macro that sets up the callee save frame to conform with 336 * Runtime::CreateCalleeSaveMethod(kSaveAllCalleeSaves) 337 */ 338.macro SETUP_SAVE_ALL_CALLEE_SAVES_FRAME 339 // art::Runtime* xIP0 = art::Runtime::instance_; 340 // Our registers aren't intermixed - just spill in order. 341 LOAD_RUNTIME_INSTANCE xIP0 342 343 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveAllCalleeSaves]; 344 ldr xIP0, [xIP0, RUNTIME_SAVE_ALL_CALLEE_SAVES_METHOD_OFFSET] 345 346 INCREASE_FRAME 176 347 348 // Ugly compile-time check, but we only have the preprocessor. 349#if (FRAME_SIZE_SAVE_ALL_CALLEE_SAVES != 176) 350#error "FRAME_SIZE_SAVE_ALL_CALLEE_SAVES(ARM64) size not as expected." 351#endif 352 353 // Stack alignment filler [sp, #8]. 354 SAVE_ALL_CALLEE_SAVES 16 355 356 // Store ArtMethod* Runtime::callee_save_methods_[kSaveAllCalleeSaves]. 357 str xIP0, [sp] 358 // Place sp in Thread::Current()->top_quick_frame. 359 mov xIP0, sp 360 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 361.endm 362 363 /* 364 * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending 365 * exception is Thread::Current()->exception_ when the runtime method frame is ready. 366 */ 367.macro DELIVER_PENDING_EXCEPTION_FRAME_READY 368 mov x0, xSELF 369 370 // Point of no return. 371 bl artDeliverPendingExceptionFromCode // artDeliverPendingExceptionFromCode(Thread*) 372 brk 0 // Unreached 373.endm 374 375 /* 376 * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending 377 * exception is Thread::Current()->exception_. 378 */ 379.macro DELIVER_PENDING_EXCEPTION 380 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME 381 DELIVER_PENDING_EXCEPTION_FRAME_READY 382.endm 383 384.macro RETURN_OR_DELIVER_PENDING_EXCEPTION_REG reg 385 ldr \reg, [xSELF, # THREAD_EXCEPTION_OFFSET] // Get exception field. 386 cbnz \reg, 1f 387 ret 3881: 389 DELIVER_PENDING_EXCEPTION 390.endm 391 392.macro RETURN_OR_DELIVER_PENDING_EXCEPTION 393 RETURN_OR_DELIVER_PENDING_EXCEPTION_REG xIP0 394.endm 395 396// Locking is needed for both managed code and JNI stubs. 397.macro LOCK_OBJECT_FAST_PATH obj, slow_lock, can_be_null 398 // Use scratch registers x8-x11 as temporaries. 399 ldr w9, [xSELF, #THREAD_ID_OFFSET] 400 .if \can_be_null 401 cbz \obj, \slow_lock 402 .endif 403 // Exclusive load/store has no immediate anymore. 404 add x8, \obj, #MIRROR_OBJECT_LOCK_WORD_OFFSET 4051: 406 ldaxr w10, [x8] // Acquire needed only in most common case. 407 eor w11, w10, w9 // Prepare the value to store if unlocked 408 // (thread id, count of 0 and preserved read barrier bits), 409 // or prepare to compare thread id for recursive lock check 410 // (lock_word.ThreadId() ^ self->ThreadId()). 411 tst w10, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED // Test the non-gc bits. 412 b.ne 2f // Check if unlocked. 413 // Unlocked case - store w11: original lock word plus thread id, preserved read barrier bits. 414 stxr w10, w11, [x8] 415 cbnz w10, 1b // If the store failed, retry. 416 ret 4172: // w10: original lock word, w9: thread id, w11: w10 ^ w9 418 // Check lock word state and thread id together, 419 tst w11, #(LOCK_WORD_STATE_MASK_SHIFTED | LOCK_WORD_THIN_LOCK_OWNER_MASK_SHIFTED) 420 b.ne \slow_lock 421 add w11, w10, #LOCK_WORD_THIN_LOCK_COUNT_ONE // Increment the recursive lock count. 422 tst w11, #LOCK_WORD_THIN_LOCK_COUNT_MASK_SHIFTED // Test the new thin lock count. 423 b.eq \slow_lock // Zero as the new count indicates overflow, go slow path. 424 stxr w10, w11, [x8] 425 cbnz w10, 1b // If the store failed, retry. 426 ret 427.endm 428 429// Unlocking is needed for both managed code and JNI stubs. 430.macro UNLOCK_OBJECT_FAST_PATH obj, slow_unlock, can_be_null 431 // Use scratch registers x8-x11 as temporaries. 432 ldr w9, [xSELF, #THREAD_ID_OFFSET] 433 .if \can_be_null 434 cbz \obj, \slow_unlock 435 .endif 436 // Exclusive load/store has no immediate anymore. 437 add x8, \obj, #MIRROR_OBJECT_LOCK_WORD_OFFSET 4381: 439#ifndef USE_READ_BARRIER 440 ldr w10, [x8] 441#else 442 ldxr w10, [x8] // Need to use atomic instructions for read barrier. 443#endif 444 eor w11, w10, w9 // Prepare the value to store if simply locked 445 // (mostly 0s, and preserved read barrier bits), 446 // or prepare to compare thread id for recursive lock check 447 // (lock_word.ThreadId() ^ self->ThreadId()). 448 tst w11, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED // Test the non-gc bits. 449 b.ne 2f // Locked recursively or by other thread? 450 // Transition to unlocked. 451#ifndef USE_READ_BARRIER 452 stlr w11, [x8] 453#else 454 stlxr w10, w11, [x8] // Need to use atomic instructions for read barrier. 455 cbnz w10, 1b // If the store failed, retry. 456#endif 457 ret 4582: 459 // Check lock word state and thread id together. 460 tst w11, #(LOCK_WORD_STATE_MASK_SHIFTED | LOCK_WORD_THIN_LOCK_OWNER_MASK_SHIFTED) 461 b.ne \slow_unlock 462 sub w11, w10, #LOCK_WORD_THIN_LOCK_COUNT_ONE // decrement count 463#ifndef USE_READ_BARRIER 464 str w11, [x8] 465#else 466 stxr w10, w11, [x8] // Need to use atomic instructions for read barrier. 467 cbnz w10, 1b // If the store failed, retry. 468#endif 469 ret 470.endm 471 472#endif // ART_RUNTIME_ARCH_ARM64_ASM_SUPPORT_ARM64_S_ 473