1%def header(): 2/* 3 * Copyright (C) 2019 The Android Open Source Project 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18/* 19 * This is a #include, not a %include, because we want the C pre-processor 20 * to expand the macros into assembler assignment statements. 21 */ 22#include "asm_support.h" 23#include "arch/x86_64/asm_support_x86_64.S" 24 25/** 26 * x86_64 ABI general notes: 27 * 28 * Caller save set: 29 * rax, rdx, rcx, rsi, rdi, r8-r11, st(0)-st(7) 30 * Callee save set: 31 * rbx, rbp, r12-r15 32 * Return regs: 33 * 32-bit in eax 34 * 64-bit in rax 35 * fp on xmm0 36 * 37 * First 8 fp parameters came in xmm0-xmm7. 38 * First 6 non-fp parameters came in rdi, rsi, rdx, rcx, r8, r9. 39 * Other parameters passed on stack, pushed right-to-left. On entry to target, first 40 * param is at 8(%esp). 41 * 42 * Stack must be 16-byte aligned to support SSE in native code. 43 */ 44 45#define IN_ARG3 %rcx 46#define IN_ARG2 %rdx 47#define IN_ARG1 %rsi 48#define IN_ARG0 %rdi 49/* Out Args */ 50#define OUT_ARG3 %rcx 51#define OUT_ARG2 %rdx 52#define OUT_ARG1 %rsi 53#define OUT_ARG0 %rdi 54#define OUT_32_ARG3 %ecx 55#define OUT_32_ARG2 %edx 56#define OUT_32_ARG1 %esi 57#define OUT_32_ARG0 %edi 58#define OUT_FP_ARG1 %xmm1 59#define OUT_FP_ARG0 %xmm0 60 61/* 62 * single-purpose registers, given names for clarity 63 */ 64#define rSELF %gs 65#define rPC %r12 66#define CFI_DEX 12 // DWARF register number of the register holding dex-pc (rPC). 67#define CFI_TMP 5 // DWARF register number of the first argument register (rdi). 68#define rFP %r13 69#define rINST %ebx 70#define rINSTq %rbx 71#define rINSTw %bx 72#define rINSTbh %bh 73#define rINSTbl %bl 74#define rIBASE %r14 75#define rREFS %r15 76#define rREFS32 %r15d 77#define CFI_REFS 15 // DWARF register number of the reference array (r15). 78 79// Temporary registers while setting up a frame. 80#define rNEW_FP %r8 81#define rNEW_REFS %r9 82#define rNEW_REFS32 %r9d 83#define CFI_NEW_REFS 9 84 85/* 86 * Get/set the 32-bit value from a Dalvik register. 87 */ 88#define VREG_ADDRESS(_vreg) (rFP,_vreg,4) 89#define VREG_HIGH_ADDRESS(_vreg) 4(rFP,_vreg,4) 90#define VREG_REF_ADDRESS(_vreg) (rREFS,_vreg,4) 91#define VREG_REF_HIGH_ADDRESS(_vreg) 4(rREFS,_vreg,4) 92 93// Includes the return address implictly pushed on stack by 'call'. 94#define CALLEE_SAVES_SIZE (6 * 8 + 4 * 8 + 1 * 8) 95 96// +8 for the ArtMethod of the caller. 97#define OFFSET_TO_FIRST_ARGUMENT_IN_STACK (CALLEE_SAVES_SIZE + 8) 98 99/* 100 * Refresh rINST. 101 * At enter to handler rINST does not contain the opcode number. 102 * However some utilities require the full value, so this macro 103 * restores the opcode number. 104 */ 105.macro REFRESH_INST _opnum 106 movb rINSTbl, rINSTbh 107 movb $$\_opnum, rINSTbl 108.endm 109 110/* 111 * Fetch the next instruction from rPC into rINSTw. Does not advance rPC. 112 */ 113.macro FETCH_INST 114 movzwq (rPC), rINSTq 115.endm 116 117/* 118 * Remove opcode from rINST, compute the address of handler and jump to it. 119 */ 120.macro GOTO_NEXT 121 movzx rINSTbl,%ecx 122 movzbl rINSTbh,rINST 123 shll MACRO_LITERAL(${handler_size_bits}), %ecx 124 addq rIBASE, %rcx 125 jmp *%rcx 126.endm 127 128/* 129 * Advance rPC by instruction count. 130 */ 131.macro ADVANCE_PC _count 132 leaq 2*\_count(rPC), rPC 133.endm 134 135/* 136 * Advance rPC by instruction count, fetch instruction and jump to handler. 137 */ 138.macro ADVANCE_PC_FETCH_AND_GOTO_NEXT _count 139 ADVANCE_PC \_count 140 FETCH_INST 141 GOTO_NEXT 142.endm 143 144.macro GET_VREG _reg _vreg 145 movl VREG_ADDRESS(\_vreg), \_reg 146.endm 147 148.macro GET_VREG_OBJECT _reg _vreg 149 movl VREG_REF_ADDRESS(\_vreg), \_reg 150.endm 151 152/* Read wide value. */ 153.macro GET_WIDE_VREG _reg _vreg 154 movq VREG_ADDRESS(\_vreg), \_reg 155.endm 156 157.macro SET_VREG _reg _vreg 158 movl \_reg, VREG_ADDRESS(\_vreg) 159 movl MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg) 160.endm 161 162/* Write wide value. reg is clobbered. */ 163.macro SET_WIDE_VREG _reg _vreg 164 movq \_reg, VREG_ADDRESS(\_vreg) 165 xorq \_reg, \_reg 166 movq \_reg, VREG_REF_ADDRESS(\_vreg) 167.endm 168 169.macro SET_VREG_OBJECT _reg _vreg 170 movl \_reg, VREG_ADDRESS(\_vreg) 171 movl \_reg, VREG_REF_ADDRESS(\_vreg) 172.endm 173 174.macro GET_VREG_HIGH _reg _vreg 175 movl VREG_HIGH_ADDRESS(\_vreg), \_reg 176.endm 177 178.macro SET_VREG_HIGH _reg _vreg 179 movl \_reg, VREG_HIGH_ADDRESS(\_vreg) 180 movl MACRO_LITERAL(0), VREG_REF_HIGH_ADDRESS(\_vreg) 181.endm 182 183.macro CLEAR_REF _vreg 184 movl MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg) 185.endm 186 187.macro CLEAR_WIDE_REF _vreg 188 movl MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg) 189 movl MACRO_LITERAL(0), VREG_REF_HIGH_ADDRESS(\_vreg) 190.endm 191 192.macro GET_VREG_XMMs _xmmreg _vreg 193 movss VREG_ADDRESS(\_vreg), \_xmmreg 194.endm 195.macro GET_VREG_XMMd _xmmreg _vreg 196 movsd VREG_ADDRESS(\_vreg), \_xmmreg 197.endm 198.macro SET_VREG_XMMs _xmmreg _vreg 199 movss \_xmmreg, VREG_ADDRESS(\_vreg) 200.endm 201.macro SET_VREG_XMMd _xmmreg _vreg 202 movsd \_xmmreg, VREG_ADDRESS(\_vreg) 203.endm 204 205// An assembly entry that has a OatQuickMethodHeader prefix. 206.macro OAT_ENTRY name, end 207 FUNCTION_TYPE(\name) 208 ASM_HIDDEN SYMBOL(\name) 209 .global SYMBOL(\name) 210 .balign 16 211 // Padding of 3 * 8 bytes to get 16 bytes alignment of code entry. 212 .long 0 213 .long 0 214 .long 0 215 // OatQuickMethodHeader. Note that the top two bits must be clear. 216 .long (SYMBOL(\end) - SYMBOL(\name)) 217SYMBOL(\name): 218.endm 219 220.macro ENTRY name 221 .text 222 ASM_HIDDEN SYMBOL(\name) 223 .global SYMBOL(\name) 224 FUNCTION_TYPE(\name) 225SYMBOL(\name): 226.endm 227 228.macro END name 229 SIZE(\name) 230.endm 231 232// Macro for defining entrypoints into runtime. We don't need to save registers 233// (we're not holding references there), but there is no 234// kDontSave runtime method. So just use the kSaveRefsOnly runtime method. 235.macro NTERP_TRAMPOLINE name, helper 236DEFINE_FUNCTION \name 237 SETUP_SAVE_REFS_ONLY_FRAME 238 call \helper 239 RESTORE_SAVE_REFS_ONLY_FRAME 240 RETURN_OR_DELIVER_PENDING_EXCEPTION 241END_FUNCTION \name 242.endm 243 244.macro CLEAR_VOLATILE_MARKER reg 245 andq MACRO_LITERAL(-2), \reg 246.endm 247 248.macro EXPORT_PC 249 movq rPC, -16(rREFS) 250.endm 251 252 253.macro BRANCH 254 // Update method counter and do a suspend check if the branch is negative. 255 testq rINSTq, rINSTq 256 js 3f 2572: 258 leaq (rPC, rINSTq, 2), rPC 259 FETCH_INST 260 GOTO_NEXT 2613: 262 movq (%rsp), %rdi 263 addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) 264 andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) 265 // If the counter overflows, handle this in the runtime. 266 jz NterpHandleHotnessOverflow 267 // Otherwise, do a suspend check. 268 testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET 269 jz 2b 270 EXPORT_PC 271 call SYMBOL(art_quick_test_suspend) 272 jmp 2b 273.endm 274 275// Expects: 276// - r10, and r11 to be available. 277// Outputs: 278// - \registers contains the dex registers size 279// - \outs contains the outs size 280// - if load_ins is 1, \ins contains the ins 281// - \code_item is replace with a pointer to the instructions 282.macro FETCH_CODE_ITEM_INFO code_item, registers, outs, ins, load_ins 283 testq MACRO_LITERAL(1), \code_item 284 je 5f 285 andq $$-2, \code_item // Remove the extra bit that marks it's a compact dex file. 286 movzwl COMPACT_CODE_ITEM_FIELDS_OFFSET(\code_item), %r10d 287 movl %r10d, \registers 288 sarl $$COMPACT_CODE_ITEM_REGISTERS_SIZE_SHIFT, \registers 289 andl $$0xf, \registers 290 movl %r10d, \outs 291 sarl $$COMPACT_CODE_ITEM_OUTS_SIZE_SHIFT, \outs 292 andl $$0xf, \outs 293 .if \load_ins 294 movl %r10d, \ins 295 sarl $$COMPACT_CODE_ITEM_INS_SIZE_SHIFT, \ins 296 andl $$0xf, \ins 297 .else 298 movl %r10d, %r11d 299 sarl $$COMPACT_CODE_ITEM_INS_SIZE_SHIFT, %r11d 300 andl $$0xf, %r11d 301 addl %r11d, \registers 302 .endif 303 testw $$COMPACT_CODE_ITEM_REGISTERS_INS_OUTS_FLAGS, COMPACT_CODE_ITEM_FLAGS_OFFSET(\code_item) 304 je 4f 305 movq \code_item, %r11 306 testw $$COMPACT_CODE_ITEM_INSNS_FLAG, COMPACT_CODE_ITEM_FLAGS_OFFSET(\code_item) 307 je 1f 308 subq $$4, %r11 3091: 310 testw $$COMPACT_CODE_ITEM_REGISTERS_FLAG, COMPACT_CODE_ITEM_FLAGS_OFFSET(\code_item) 311 je 2f 312 subq $$2, %r11 313 movzwl (%r11), %r10d 314 addl %r10d, \registers 3152: 316 testw $$COMPACT_CODE_ITEM_INS_FLAG, COMPACT_CODE_ITEM_FLAGS_OFFSET(\code_item) 317 je 3f 318 subq $$2, %r11 319 movzwl (%r11), %r10d 320 .if \load_ins 321 addl %r10d, \ins 322 .else 323 addl %r10d, \registers 324 .endif 3253: 326 testw $$COMPACT_CODE_ITEM_OUTS_FLAG, COMPACT_CODE_ITEM_FLAGS_OFFSET(\code_item) 327 je 4f 328 subq $$2, %r11 329 movzwl (%r11), %r10d 330 addl %r10d, \outs 3314: 332 .if \load_ins 333 addl \ins, \registers 334 .endif 335 addq $$COMPACT_CODE_ITEM_INSNS_OFFSET, \code_item 336 jmp 6f 3375: 338 // Fetch dex register size. 339 movzwl CODE_ITEM_REGISTERS_SIZE_OFFSET(\code_item), \registers 340 // Fetch outs size. 341 movzwl CODE_ITEM_OUTS_SIZE_OFFSET(\code_item), \outs 342 .if \load_ins 343 movzwl CODE_ITEM_INS_SIZE_OFFSET(\code_item), \ins 344 .endif 345 addq $$CODE_ITEM_INSNS_OFFSET, \code_item 3466: 347.endm 348 349// Setup the stack to start executing the method. Expects: 350// - rdi to contain the ArtMethod 351// - rbx, r10, r11 to be available. 352// 353// Outputs 354// - rbx contains the dex registers size 355// - r11 contains the old stack pointer. 356// - \code_item is replace with a pointer to the instructions 357// - if load_ins is 1, r14 contains the ins 358.macro SETUP_STACK_FRAME code_item, refs, refs32, fp, cfi_refs, load_ins 359 FETCH_CODE_ITEM_INFO \code_item, %ebx, \refs32, %r14d, \load_ins 360 361 // Compute required frame size for dex registers: ((2 * ebx) + refs) 362 leaq (\refs, %rbx, 2), %r11 363 salq $$2, %r11 364 365 // Compute new stack pointer in r10: add 24 for saving the previous frame, 366 // pc, and method being executed. 367 leaq -24(%rsp), %r10 368 subq %r11, %r10 369 // Alignment 370 // Note: There may be two pieces of alignment but there is no need to align 371 // out args to `kPointerSize` separately before aligning to kStackAlignment. 372 andq $$-16, %r10 373 374 // Set reference and dex registers, align to pointer size for previous frame and dex pc. 375 leaq 24 + 4(%r10, \refs, 4), \refs 376 andq LITERAL(-__SIZEOF_POINTER__), \refs 377 leaq (\refs, %rbx, 4), \fp 378 379 // Now setup the stack pointer. 380 movq %rsp, %r11 381 CFI_DEF_CFA_REGISTER(r11) 382 movq %r10, %rsp 383 movq %r11, -8(\refs) 384 CFI_DEF_CFA_BREG_PLUS_UCONST \cfi_refs, -8, ((6 + 4 + 1) * 8) 385 386 // Put nulls in reference frame. 387 testl %ebx, %ebx 388 je 2f 389 movq \refs, %r10 3901: 391 movl $$0, (%r10) 392 addq $$4, %r10 393 cmpq %r10, \fp 394 jne 1b 3952: 396 // Save the ArtMethod. 397 movq %rdi, (%rsp) 398.endm 399 400// Puts the next floating point argument into the expected register, 401// fetching values based on a non-range invoke. 402// Uses rax as temporary. 403// 404// TODO: We could simplify a lot of code by loading the G argument into 405// the "inst" register. Given that we enter the handler with "1(rPC)" in 406// the rINST, we can just add rINST<<16 to the args and we don't even 407// need to pass "arg_index" around. 408.macro LOOP_OVER_SHORTY_LOADING_XMMS xmm_reg, inst, shorty, arg_index, finished 4091: // LOOP 410 movb (REG_VAR(shorty)), %al // bl := *shorty 411 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 412 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 413 je VAR(finished) 414 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 415 je 2f 416 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 417 je 3f 418 shrq MACRO_LITERAL(4), REG_VAR(inst) 419 addq MACRO_LITERAL(1), REG_VAR(arg_index) 420 // Handle extra argument in arg array taken by a long. 421 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 422 jne 1b 423 shrq MACRO_LITERAL(4), REG_VAR(inst) 424 addq MACRO_LITERAL(1), REG_VAR(arg_index) 425 jmp 1b // goto LOOP 4262: // FOUND_DOUBLE 427 subq MACRO_LITERAL(8), %rsp 428 movq REG_VAR(inst), %rax 429 andq MACRO_LITERAL(0xf), %rax 430 GET_VREG %eax, %rax 431 movl %eax, (%rsp) 432 shrq MACRO_LITERAL(4), REG_VAR(inst) 433 addq MACRO_LITERAL(1), REG_VAR(arg_index) 434 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 435 je 5f 436 movq REG_VAR(inst), %rax 437 andq MACRO_LITERAL(0xf), %rax 438 shrq MACRO_LITERAL(4), REG_VAR(inst) 439 addq MACRO_LITERAL(1), REG_VAR(arg_index) 440 jmp 6f 4415: 442 movzbl 1(rPC), %eax 443 andq MACRO_LITERAL(0xf), %rax 4446: 445 GET_VREG %eax, %rax 446 movl %eax, 4(%rsp) 447 movsd (%rsp), REG_VAR(xmm_reg) 448 addq MACRO_LITERAL(8), %rsp 449 jmp 4f 4503: // FOUND_FLOAT 451 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 452 je 7f 453 movq REG_VAR(inst), %rax 454 andq MACRO_LITERAL(0xf), %rax 455 shrq MACRO_LITERAL(4), REG_VAR(inst) 456 addq MACRO_LITERAL(1), REG_VAR(arg_index) 457 jmp 8f 4587: 459 movzbl 1(rPC), %eax 460 andq MACRO_LITERAL(0xf), %rax 4618: 462 GET_VREG_XMMs REG_VAR(xmm_reg), %rax 4634: 464.endm 465 466// Puts the next int/long/object argument in the expected register, 467// fetching values based on a non-range invoke. 468// Uses rax as temporary. 469.macro LOOP_OVER_SHORTY_LOADING_GPRS gpr_reg64, gpr_reg32, inst, shorty, arg_index, finished 4701: // LOOP 471 movb (REG_VAR(shorty)), %al // al := *shorty 472 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 473 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 474 je VAR(finished) 475 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 476 je 2f 477 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 478 je 3f 479 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 480 je 4f 481 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 482 je 7f 483 movq REG_VAR(inst), %rax 484 andq MACRO_LITERAL(0xf), %rax 485 shrq MACRO_LITERAL(4), REG_VAR(inst) 486 addq MACRO_LITERAL(1), REG_VAR(arg_index) 487 jmp 8f 4887: 489 movzbl 1(rPC), %eax 490 andq MACRO_LITERAL(0xf), %rax 4918: 492 GET_VREG REG_VAR(gpr_reg32), %rax 493 jmp 5f 4942: // FOUND_LONG 495 subq MACRO_LITERAL(8), %rsp 496 movq REG_VAR(inst), %rax 497 andq MACRO_LITERAL(0xf), %rax 498 GET_VREG %eax, %rax 499 movl %eax, (%rsp) 500 shrq MACRO_LITERAL(4), REG_VAR(inst) 501 addq MACRO_LITERAL(1), REG_VAR(arg_index) 502 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 503 je 9f 504 movq REG_VAR(inst), %rax 505 andq MACRO_LITERAL(0xf), %rax 506 shrq MACRO_LITERAL(4), REG_VAR(inst) 507 addq MACRO_LITERAL(1), REG_VAR(arg_index) 508 jmp 10f 5099: 510 movzbl 1(rPC), %eax 511 andq MACRO_LITERAL(0xf), %rax 51210: 513 GET_VREG %eax, %rax 514 movl %eax, 4(%rsp) 515 movq (%rsp), REG_VAR(gpr_reg64) 516 addq MACRO_LITERAL(8), %rsp 517 jmp 5f 5183: // SKIP_FLOAT 519 shrq MACRO_LITERAL(4), REG_VAR(inst) 520 addq MACRO_LITERAL(1), REG_VAR(arg_index) 521 jmp 1b 5224: // SKIP_DOUBLE 523 shrq MACRO_LITERAL(4), REG_VAR(inst) 524 addq MACRO_LITERAL(1), REG_VAR(arg_index) 525 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 526 je 1b 527 shrq MACRO_LITERAL(4), REG_VAR(inst) 528 addq MACRO_LITERAL(1), REG_VAR(arg_index) 529 jmp 1b 5305: 531.endm 532 533// Puts the next floating point argument into the expected register, 534// fetching values based on a range invoke. 535// Uses rax as temporary. 536.macro LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm_reg, shorty, arg_index, stack_index, finished 5371: // LOOP 538 movb (REG_VAR(shorty)), %al // al := *shorty 539 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 540 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 541 je VAR(finished) 542 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 543 je 2f 544 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 545 je 3f 546 addq MACRO_LITERAL(1), REG_VAR(arg_index) 547 addq MACRO_LITERAL(1), REG_VAR(stack_index) 548 // Handle extra argument in arg array taken by a long. 549 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 550 jne 1b 551 addq MACRO_LITERAL(1), REG_VAR(arg_index) 552 addq MACRO_LITERAL(1), REG_VAR(stack_index) 553 jmp 1b // goto LOOP 5542: // FOUND_DOUBLE 555 GET_VREG_XMMd REG_VAR(xmm_reg), REG_VAR(arg_index) 556 addq MACRO_LITERAL(2), REG_VAR(arg_index) 557 addq MACRO_LITERAL(2), REG_VAR(stack_index) 558 jmp 4f 5593: // FOUND_FLOAT 560 GET_VREG_XMMs REG_VAR(xmm_reg), REG_VAR(arg_index) 561 addq MACRO_LITERAL(1), REG_VAR(arg_index) 562 addq MACRO_LITERAL(1), REG_VAR(stack_index) 5634: 564.endm 565 566// Puts the next floating point argument into the expected stack slot, 567// fetching values based on a range invoke. 568// Uses rax as temporary. 569// 570// TODO: We could just copy all the vregs to the stack slots in a simple loop 571// (or REP MOVSD) without looking at the shorty at all. (We could also drop 572// the "stack_index" from the macros for loading registers.) We could also do 573// that conditionally if argument word count > 6; otherwise we know that all 574// args fit into registers. 575.macro LOOP_RANGE_OVER_FPs shorty, arg_index, stack_index, finished 5761: // LOOP 577 movb (REG_VAR(shorty)), %al // bl := *shorty 578 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 579 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 580 je VAR(finished) 581 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 582 je 2f 583 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 584 je 3f 585 addq MACRO_LITERAL(1), REG_VAR(arg_index) 586 addq MACRO_LITERAL(1), REG_VAR(stack_index) 587 // Handle extra argument in arg array taken by a long. 588 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 589 jne 1b 590 addq MACRO_LITERAL(1), REG_VAR(arg_index) 591 addq MACRO_LITERAL(1), REG_VAR(stack_index) 592 jmp 1b // goto LOOP 5932: // FOUND_DOUBLE 594 movq (rFP, REG_VAR(arg_index), 4), %rax 595 movq %rax, 8(%rsp, REG_VAR(stack_index), 4) 596 addq MACRO_LITERAL(2), REG_VAR(arg_index) 597 addq MACRO_LITERAL(2), REG_VAR(stack_index) 598 jmp 1b 5993: // FOUND_FLOAT 600 movl (rFP, REG_VAR(arg_index), 4), %eax 601 movl %eax, 8(%rsp, REG_VAR(stack_index), 4) 602 addq MACRO_LITERAL(1), REG_VAR(arg_index) 603 addq MACRO_LITERAL(1), REG_VAR(stack_index) 604 jmp 1b 605.endm 606 607// Puts the next int/long/object argument in the expected register, 608// fetching values based on a range invoke. 609// Uses rax as temporary. 610.macro LOOP_RANGE_OVER_SHORTY_LOADING_GPRS gpr_reg64, gpr_reg32, shorty, arg_index, stack_index, finished 6111: // LOOP 612 movb (REG_VAR(shorty)), %al // al := *shorty 613 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 614 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 615 je VAR(finished) 616 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 617 je 2f 618 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 619 je 3f 620 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 621 je 4f 622 movl (rFP, REG_VAR(arg_index), 4), REG_VAR(gpr_reg32) 623 addq MACRO_LITERAL(1), REG_VAR(arg_index) 624 addq MACRO_LITERAL(1), REG_VAR(stack_index) 625 jmp 5f 6262: // FOUND_LONG 627 movq (rFP, REG_VAR(arg_index), 4), REG_VAR(gpr_reg64) 628 addq MACRO_LITERAL(2), REG_VAR(arg_index) 629 addq MACRO_LITERAL(2), REG_VAR(stack_index) 630 jmp 5f 6313: // SKIP_FLOAT 632 addq MACRO_LITERAL(1), REG_VAR(arg_index) 633 addq MACRO_LITERAL(1), REG_VAR(stack_index) 634 jmp 1b 6354: // SKIP_DOUBLE 636 addq MACRO_LITERAL(2), REG_VAR(arg_index) 637 addq MACRO_LITERAL(2), REG_VAR(stack_index) 638 jmp 1b 6395: 640.endm 641 642// Puts the next int/long/object argument in the expected stack slot, 643// fetching values based on a range invoke. 644// Uses rax as temporary. 645.macro LOOP_RANGE_OVER_INTs shorty, arg_index, stack_index, finished 6461: // LOOP 647 movb (REG_VAR(shorty)), %al // al := *shorty 648 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 649 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 650 je VAR(finished) 651 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 652 je 2f 653 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 654 je 3f 655 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 656 je 4f 657 movl (rFP, REG_VAR(arg_index), 4), %eax 658 movl %eax, 8(%rsp, REG_VAR(stack_index), 4) 659 addq MACRO_LITERAL(1), REG_VAR(arg_index) 660 addq MACRO_LITERAL(1), REG_VAR(stack_index) 661 jmp 1b 6622: // FOUND_LONG 663 movq (rFP, REG_VAR(arg_index), 4), %rax 664 movq %rax, 8(%rsp, REG_VAR(stack_index), 4) 665 addq MACRO_LITERAL(2), REG_VAR(arg_index) 666 addq MACRO_LITERAL(2), REG_VAR(stack_index) 667 jmp 1b 6683: // SKIP_FLOAT 669 addq MACRO_LITERAL(1), REG_VAR(arg_index) 670 addq MACRO_LITERAL(1), REG_VAR(stack_index) 671 jmp 1b 6724: // SKIP_DOUBLE 673 addq MACRO_LITERAL(2), REG_VAR(arg_index) 674 addq MACRO_LITERAL(2), REG_VAR(stack_index) 675 jmp 1b 676.endm 677 678// Puts the next floating point parameter passed in physical register 679// in the expected dex register array entry. 680// Uses rax as temporary. 681.macro LOOP_OVER_SHORTY_STORING_XMMS xmm_reg, shorty, arg_index, fp, finished 6821: // LOOP 683 movb (REG_VAR(shorty)), %al // al := *shorty 684 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 685 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 686 je VAR(finished) 687 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 688 je 2f 689 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 690 je 3f 691 addq MACRO_LITERAL(1), REG_VAR(arg_index) 692 // Handle extra argument in arg array taken by a long. 693 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 694 jne 1b 695 addq MACRO_LITERAL(1), REG_VAR(arg_index) 696 jmp 1b // goto LOOP 6972: // FOUND_DOUBLE 698 movsd REG_VAR(xmm_reg),(REG_VAR(fp), REG_VAR(arg_index), 4) 699 addq MACRO_LITERAL(2), REG_VAR(arg_index) 700 jmp 4f 7013: // FOUND_FLOAT 702 movss REG_VAR(xmm_reg), (REG_VAR(fp), REG_VAR(arg_index), 4) 703 addq MACRO_LITERAL(1), REG_VAR(arg_index) 7044: 705.endm 706 707// Puts the next int/long/object parameter passed in physical register 708// in the expected dex register array entry, and in case of object in the 709// expected reference array entry. 710// Uses rax as temporary. 711.macro LOOP_OVER_SHORTY_STORING_GPRS gpr_reg64, gpr_reg32, shorty, arg_index, regs, refs, finished 7121: // LOOP 713 movb (REG_VAR(shorty)), %al // al := *shorty 714 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 715 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 716 je VAR(finished) 717 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 718 je 2f 719 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 720 je 3f 721 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 722 je 4f 723 movl REG_VAR(gpr_reg32), (REG_VAR(regs), REG_VAR(arg_index), 4) 724 cmpb MACRO_LITERAL(76), %al // if (al != 'L') goto NOT_REFERENCE 725 jne 6f 726 movl REG_VAR(gpr_reg32), (REG_VAR(refs), REG_VAR(arg_index), 4) 7276: // NOT_REFERENCE 728 addq MACRO_LITERAL(1), REG_VAR(arg_index) 729 jmp 5f 7302: // FOUND_LONG 731 movq REG_VAR(gpr_reg64), (REG_VAR(regs), REG_VAR(arg_index), 4) 732 addq MACRO_LITERAL(2), REG_VAR(arg_index) 733 jmp 5f 7343: // SKIP_FLOAT 735 addq MACRO_LITERAL(1), REG_VAR(arg_index) 736 jmp 1b 7374: // SKIP_DOUBLE 738 addq MACRO_LITERAL(2), REG_VAR(arg_index) 739 jmp 1b 7405: 741.endm 742 743// Puts the next floating point parameter passed in stack 744// in the expected dex register array entry. 745// Uses rax as temporary. 746// 747// TODO: Or we could just spill regs to the reserved slots in the caller's 748// frame and copy all regs in a simple loop. This time, however, we would 749// need to look at the shorty anyway to look for the references. 750// (The trade-off is different for passing arguments and receiving them.) 751.macro LOOP_OVER_FPs shorty, arg_index, regs, stack_ptr, finished 7521: // LOOP 753 movb (REG_VAR(shorty)), %al // al := *shorty 754 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 755 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 756 je VAR(finished) 757 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 758 je 2f 759 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 760 je 3f 761 addq MACRO_LITERAL(1), REG_VAR(arg_index) 762 // Handle extra argument in arg array taken by a long. 763 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 764 jne 1b 765 addq MACRO_LITERAL(1), REG_VAR(arg_index) 766 jmp 1b // goto LOOP 7672: // FOUND_DOUBLE 768 movq OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %rax 769 movq %rax, (REG_VAR(regs), REG_VAR(arg_index), 4) 770 addq MACRO_LITERAL(2), REG_VAR(arg_index) 771 jmp 1b 7723: // FOUND_FLOAT 773 movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %eax 774 movl %eax, (REG_VAR(regs), REG_VAR(arg_index), 4) 775 addq MACRO_LITERAL(1), REG_VAR(arg_index) 776 jmp 1b 777.endm 778 779// Puts the next int/long/object parameter passed in stack 780// in the expected dex register array entry, and in case of object in the 781// expected reference array entry. 782// Uses rax as temporary. 783.macro LOOP_OVER_INTs shorty, arg_index, regs, refs, stack_ptr, finished 7841: // LOOP 785 movb (REG_VAR(shorty)), %al // al := *shorty 786 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 787 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 788 je VAR(finished) 789 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 790 je 2f 791 cmpb MACRO_LITERAL(76), %al // if (al == 'L') goto FOUND_REFERENCE 792 je 6f 793 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 794 je 3f 795 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 796 je 4f 797 movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %eax 798 movl %eax, (REG_VAR(regs), REG_VAR(arg_index), 4) 799 addq MACRO_LITERAL(1), REG_VAR(arg_index) 800 jmp 1b 8016: // FOUND_REFERENCE 802 movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %eax 803 movl %eax, (REG_VAR(regs), REG_VAR(arg_index), 4) 804 movl %eax, (REG_VAR(refs), REG_VAR(arg_index), 4) 805 addq MACRO_LITERAL(1), REG_VAR(arg_index) 806 jmp 1b 8072: // FOUND_LONG 808 movq OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %rax 809 movq %rax, (REG_VAR(regs), REG_VAR(arg_index), 4) 810 addq MACRO_LITERAL(2), REG_VAR(arg_index) 811 jmp 1b 8123: // SKIP_FLOAT 813 addq MACRO_LITERAL(1), REG_VAR(arg_index) 814 jmp 1b 8154: // SKIP_DOUBLE 816 addq MACRO_LITERAL(2), REG_VAR(arg_index) 817 jmp 1b 818.endm 819 820// Increase method hotness and do suspend check before starting executing the method. 821.macro START_EXECUTING_INSTRUCTIONS 822 movq (%rsp), %rdi 823 addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) 824 andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) 825 jz 2f 826 testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET 827 jz 1f 828 EXPORT_PC 829 call SYMBOL(art_quick_test_suspend) 8301: 831 FETCH_INST 832 GOTO_NEXT 8332: 834 movq $$0, %rsi 835 movq rFP, %rdx 836 call nterp_hot_method 837 jmp 1b 838.endm 839 840.macro SPILL_ALL_CALLEE_SAVES 841 PUSH r15 842 PUSH r14 843 PUSH r13 844 PUSH r12 845 PUSH rbp 846 PUSH rbx 847 SETUP_FP_CALLEE_SAVE_FRAME 848.endm 849 850.macro RESTORE_ALL_CALLEE_SAVES 851 RESTORE_FP_CALLEE_SAVE_FRAME 852 POP rbx 853 POP rbp 854 POP r12 855 POP r13 856 POP r14 857 POP r15 858.endm 859 860// Helper to setup the stack after doing a nterp to nterp call. This will setup: 861// - rNEW_FP: the new pointer to dex registers 862// - rNEW_REFS: the new pointer to references 863// - rPC: the new PC pointer to execute 864// - edi: number of arguments 865// - ecx: first dex register 866// 867// This helper expects: 868// - rax to contain the code item 869.macro SETUP_STACK_FOR_INVOKE 870 // We do the same stack overflow check as the compiler. See CanMethodUseNterp 871 // in how we limit the maximum nterp frame size. 872 testq %rax, -STACK_OVERFLOW_RESERVED_BYTES(%rsp) 873 874 // Spill all callee saves to have a consistent stack frame whether we 875 // are called by compiled code or nterp. 876 SPILL_ALL_CALLEE_SAVES 877 878 // Setup the frame. 879 SETUP_STACK_FRAME %rax, rNEW_REFS, rNEW_REFS32, rNEW_FP, CFI_NEW_REFS, load_ins=0 880 // Make r11 point to the top of the dex register array. 881 leaq (rNEW_FP, %rbx, 4), %r11 882 883 // Fetch instruction information before replacing rPC. 884 movzbl 1(rPC), %edi 885 movzwl 4(rPC), %ecx 886 887 // Set the dex pc pointer. 888 movq %rax, rPC 889 CFI_DEFINE_DEX_PC_WITH_OFFSET(CFI_TMP, CFI_DEX, 0) 890.endm 891 892// Setup arguments based on a non-range nterp to nterp call, and start executing 893// the method. We expect: 894// - rNEW_FP: the new pointer to dex registers 895// - rNEW_REFS: the new pointer to references 896// - rPC: the new PC pointer to execute 897// - edi: number of arguments 898// - ecx: first dex register 899// - r11: top of dex register array 900// - esi: receiver if non-static. 901.macro SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 902 // Now all temporary registers (except r11 containing top of registers array) 903 // are available, copy the parameters. 904 // /* op vA, vB, {vC...vG} */ 905 movl %edi, %eax 906 shrl $$4, %eax # Number of arguments 907 jz 6f # shl sets the Z flag 908 movq MACRO_LITERAL(-1), %r10 909 cmpl MACRO_LITERAL(2), %eax 910 jl 1f 911 je 2f 912 cmpl MACRO_LITERAL(4), %eax 913 jl 3f 914 je 4f 915 916 // We use a decrementing r10 to store references relative 917 // to rNEW_FP and dex registers relative to r11. 918 // 919 // TODO: We could set up r10 as the number of registers (this can be an additional output from 920 // SETUP_STACK_FOR_INVOKE) and then just decrement it by one before copying each arg to 921 // (rNEW_FP, r10, 4) and (rNEW_REFS, r10, 4). 922 // Maybe even introduce macros NEW_VREG_ADDRESS/NEW_VREG_REF_ADDRESS. 9235: 924 andq MACRO_LITERAL(15), %rdi 925 GET_VREG_OBJECT %edx, %rdi 926 movl %edx, (rNEW_FP, %r10, 4) 927 GET_VREG %edx, %rdi 928 movl %edx, (%r11, %r10, 4) 929 subq MACRO_LITERAL(1), %r10 9304: 931 movl %ecx, %eax 932 shrl MACRO_LITERAL(12), %eax 933 GET_VREG_OBJECT %edx, %rax 934 movl %edx, (rNEW_FP, %r10, 4) 935 GET_VREG %edx, %rax 936 movl %edx, (%r11, %r10, 4) 937 subq MACRO_LITERAL(1), %r10 9383: 939 movl %ecx, %eax 940 shrl MACRO_LITERAL(8), %eax 941 andl MACRO_LITERAL(0xf), %eax 942 GET_VREG_OBJECT %edx, %rax 943 movl %edx, (rNEW_FP, %r10, 4) 944 GET_VREG %edx, %rax 945 movl %edx, (%r11, %r10, 4) 946 subq MACRO_LITERAL(1), %r10 9472: 948 movl %ecx, %eax 949 shrl MACRO_LITERAL(4), %eax 950 andl MACRO_LITERAL(0xf), %eax 951 GET_VREG_OBJECT %edx, %rax 952 movl %edx, (rNEW_FP, %r10, 4) 953 GET_VREG %edx, %rax 954 movl %edx, (%r11, %r10, 4) 955 subq MACRO_LITERAL(1), %r10 9561: 957 .if \is_string_init 958 // Ignore the first argument 959 .elseif \is_static 960 movl %ecx, %eax 961 andq MACRO_LITERAL(0x000f), %rax 962 GET_VREG_OBJECT %edx, %rax 963 movl %edx, (rNEW_FP, %r10, 4) 964 GET_VREG %edx, %rax 965 movl %edx, (%r11, %r10, 4) 966 .else 967 movl %esi, (rNEW_FP, %r10, 4) 968 movl %esi, (%r11, %r10, 4) 969 .endif 970 9716: 972 // Start executing the method. 973 movq rNEW_FP, rFP 974 movq rNEW_REFS, rREFS 975 CFI_DEF_CFA_BREG_PLUS_UCONST CFI_REFS, -8, ((6 + 4 + 1) * 8) 976 START_EXECUTING_INSTRUCTIONS 977.endm 978 979// Setup arguments based on a range nterp to nterp call, and start executing 980// the method. 981.macro SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 982 // edi is number of arguments 983 // ecx is first register 984 movq MACRO_LITERAL(-4), %r10 985 .if \is_string_init 986 // Ignore the first argument 987 subl $$1, %edi 988 addl $$1, %ecx 989 .elseif !\is_static 990 subl $$1, %edi 991 addl $$1, %ecx 992 .endif 993 994 testl %edi, %edi 995 je 2f 996 leaq (rREFS, %rcx, 4), %rax # pointer to first argument in reference array 997 leaq (%rax, %rdi, 4), %rax # pointer to last argument in reference array 998 leaq (rFP, %rcx, 4), %rcx # pointer to first argument in register array 999 leaq (%rcx, %rdi, 4), %rdi # pointer to last argument in register array 1000 // TODO: Same comment for copying arguments as in SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE. 10011: 1002 movl -4(%rax), %edx 1003 movl %edx, (rNEW_FP, %r10, 1) 1004 movl -4(%rdi), %edx 1005 movl %edx, (%r11, %r10, 1) 1006 subq MACRO_LITERAL(4), %r10 1007 subq MACRO_LITERAL(4), %rax 1008 subq MACRO_LITERAL(4), %rdi 1009 cmpq %rcx, %rdi 1010 jne 1b 1011 10122: 1013 .if \is_string_init 1014 // Ignore first argument 1015 .elseif !\is_static 1016 movl %esi, (rNEW_FP, %r10, 1) 1017 movl %esi, (%r11, %r10, 1) 1018 .endif 1019 movq rNEW_FP, rFP 1020 movq rNEW_REFS, rREFS 1021 CFI_DEF_CFA_BREG_PLUS_UCONST CFI_REFS, -8, ((6 + 4 + 1) * 8) 1022 START_EXECUTING_INSTRUCTIONS 1023.endm 1024 1025.macro GET_SHORTY dest, is_interface, is_polymorphic, is_custom 1026 push %rdi 1027 push %rsi 1028 .if \is_polymorphic 1029 movq 16(%rsp), %rdi 1030 movq rPC, %rsi 1031 call SYMBOL(NterpGetShortyFromInvokePolymorphic) 1032 .elseif \is_custom 1033 movq 16(%rsp), %rdi 1034 movq rPC, %rsi 1035 call SYMBOL(NterpGetShortyFromInvokeCustom) 1036 .elseif \is_interface 1037 movq 16(%rsp), %rdi 1038 movzwl 2(rPC), %esi 1039 call SYMBOL(NterpGetShortyFromMethodId) 1040 .else 1041 call SYMBOL(NterpGetShorty) 1042 .endif 1043 pop %rsi 1044 pop %rdi 1045 movq %rax, \dest 1046.endm 1047 1048.macro GET_SHORTY_SLOW_PATH dest, is_interface 1049 // Save all registers that can hold arguments in the fast path. 1050 push %rdi 1051 push %rsi 1052 push %rdx 1053 subq MACRO_LITERAL(8), %rsp 1054 mov %xmm0, (%rsp) 1055 .if \is_interface 1056 movq 32(%rsp), %rdi 1057 movzwl 2(rPC), %esi 1058 call SYMBOL(NterpGetShortyFromMethodId) 1059 .else 1060 call SYMBOL(NterpGetShorty) 1061 .endif 1062 mov (%rsp), %xmm0 1063 addq MACRO_LITERAL(8), %rsp 1064 pop %rdx 1065 pop %rsi 1066 pop %rdi 1067 movq %rax, \dest 1068.endm 1069 1070// Uses r9 as temporary. 1071.macro DO_ENTRY_POINT_CHECK call_compiled_code 1072 // On entry, the method is %rdi, the instance is %rsi 1073 leaq ExecuteNterpImpl(%rip), %r9 1074 cmpq %r9, ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) 1075 jne VAR(call_compiled_code) 1076 1077 movq ART_METHOD_DATA_OFFSET_64(%rdi), %rax 1078.endm 1079 1080// Uses r9 and r10 as temporary 1081.macro UPDATE_REGISTERS_FOR_STRING_INIT old_value, new_value 1082 movq rREFS, %r9 1083 movq rFP, %r10 10841: 1085 cmpl (%r9), \old_value 1086 jne 2f 1087 movl \new_value, (%r9) 1088 movl \new_value, (%r10) 10892: 1090 addq $$4, %r9 1091 addq $$4, %r10 1092 cmpq %r9, rFP 1093 jne 1b 1094.endm 1095 1096.macro COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, suffix="", is_string_init=0, is_polymorphic=0, is_custom=0 1097 .if \is_polymorphic 1098 // We always go to compiled code for polymorphic calls. 1099 .elseif \is_custom 1100 // We always go to compiled code for custom calls. 1101 .else 1102 DO_ENTRY_POINT_CHECK .Lcall_compiled_code_\suffix 1103 .if \is_string_init 1104 call nterp_to_nterp_string_init_non_range 1105 .elseif \is_static 1106 call nterp_to_nterp_static_non_range 1107 .else 1108 call nterp_to_nterp_instance_non_range 1109 .endif 1110 jmp .Ldone_return_\suffix 1111 .endif 1112 1113.Lcall_compiled_code_\suffix: 1114 .if \is_polymorphic 1115 // No fast path for polymorphic calls. 1116 .elseif \is_custom 1117 // No fast path for custom calls. 1118 .elseif \is_string_init 1119 // No fast path for string.init. 1120 .else 1121 testl $$ART_METHOD_NTERP_INVOKE_FAST_PATH_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi) 1122 je .Lfast_path_with_few_args_\suffix 1123 movzbl 1(rPC), %r9d 1124 movl %r9d, %ebp 1125 shrl MACRO_LITERAL(4), %ebp # Number of arguments 1126 .if \is_static 1127 jz .Linvoke_fast_path_\suffix # shl sets the Z flag 1128 .else 1129 cmpl MACRO_LITERAL(1), %ebp 1130 je .Linvoke_fast_path_\suffix 1131 .endif 1132 movzwl 4(rPC), %r11d 1133 cmpl MACRO_LITERAL(2), %ebp 1134 .if \is_static 1135 jl .Lone_arg_fast_path_\suffix 1136 .endif 1137 je .Ltwo_args_fast_path_\suffix 1138 cmpl MACRO_LITERAL(4), %ebp 1139 jl .Lthree_args_fast_path_\suffix 1140 je .Lfour_args_fast_path_\suffix 1141 1142 andl MACRO_LITERAL(0xf), %r9d 1143 GET_VREG %r9d, %r9 1144.Lfour_args_fast_path_\suffix: 1145 movl %r11d, %r8d 1146 shrl MACRO_LITERAL(12), %r8d 1147 GET_VREG %r8d, %r8 1148.Lthree_args_fast_path_\suffix: 1149 movl %r11d, %ecx 1150 shrl MACRO_LITERAL(8), %ecx 1151 andl MACRO_LITERAL(0xf), %ecx 1152 GET_VREG %ecx, %rcx 1153.Ltwo_args_fast_path_\suffix: 1154 movl %r11d, %edx 1155 shrl MACRO_LITERAL(4), %edx 1156 andl MACRO_LITERAL(0xf), %edx 1157 GET_VREG %edx, %rdx 1158.Lone_arg_fast_path_\suffix: 1159 .if \is_static 1160 andl MACRO_LITERAL(0xf), %r11d 1161 GET_VREG %esi, %r11 1162 .else 1163 // First argument already in %esi. 1164 .endif 1165.Linvoke_fast_path_\suffix: 1166 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 1167 ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 1168 1169.Lfast_path_with_few_args_\suffix: 1170 // Fast path when we have zero or one argument (modulo 'this'). If there 1171 // is one argument, we can put it in both floating point and core register. 1172 movzbl 1(rPC), %r9d 1173 shrl MACRO_LITERAL(4), %r9d # Number of arguments 1174 .if \is_static 1175 cmpl MACRO_LITERAL(1), %r9d 1176 jl .Linvoke_with_few_args_\suffix 1177 jne .Lget_shorty_\suffix 1178 movzwl 4(rPC), %r9d 1179 andl MACRO_LITERAL(0xf), %r9d // dex register of first argument 1180 GET_VREG %esi, %r9 1181 movd %esi, %xmm0 1182 .else 1183 cmpl MACRO_LITERAL(2), %r9d 1184 jl .Linvoke_with_few_args_\suffix 1185 jne .Lget_shorty_\suffix 1186 movzwl 4(rPC), %r9d 1187 shrl MACRO_LITERAL(4), %r9d 1188 andl MACRO_LITERAL(0xf), %r9d // dex register of second argument 1189 GET_VREG %edx, %r9 1190 movd %edx, %xmm0 1191 .endif 1192.Linvoke_with_few_args_\suffix: 1193 // Check if the next instruction is move-result or move-result-wide. 1194 // If it is, we fetch the shorty and jump to the regular invocation. 1195 movzwq 6(rPC), %r9 1196 andl MACRO_LITERAL(0xfe), %r9d 1197 cmpl MACRO_LITERAL(0x0a), %r9d 1198 je .Lget_shorty_and_invoke_\suffix 1199 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 1200 ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 1201.Lget_shorty_and_invoke_\suffix: 1202 .if \is_interface 1203 // Save interface method, used for conflict resolution, in a callee-save register. 1204 movq %rax, %xmm12 1205 .endif 1206 GET_SHORTY_SLOW_PATH rINSTq, \is_interface 1207 jmp .Lgpr_setup_finished_\suffix 1208 .endif 1209 1210.Lget_shorty_\suffix: 1211 .if \is_interface 1212 // Save interface method, used for conflict resolution, in a callee-save register. 1213 movq %rax, %xmm12 1214 .endif 1215 GET_SHORTY rINSTq, \is_interface, \is_polymorphic, \is_custom 1216 // From this point: 1217 // - rISNTq contains shorty (in callee-save to switch over return value after call). 1218 // - rdi contains method 1219 // - rsi contains 'this' pointer for instance method. 1220 leaq 1(rINSTq), %r9 // shorty + 1 ; ie skip return arg character 1221 movzwl 4(rPC), %r11d // arguments 1222 .if \is_string_init 1223 shrq MACRO_LITERAL(4), %r11 1224 movq $$1, %r10 // ignore first argument 1225 .elseif \is_static 1226 movq $$0, %r10 // arg_index 1227 .else 1228 shrq MACRO_LITERAL(4), %r11 1229 movq $$1, %r10 // arg_index 1230 .endif 1231 LOOP_OVER_SHORTY_LOADING_XMMS xmm0, r11, r9, r10, .Lxmm_setup_finished_\suffix 1232 LOOP_OVER_SHORTY_LOADING_XMMS xmm1, r11, r9, r10, .Lxmm_setup_finished_\suffix 1233 LOOP_OVER_SHORTY_LOADING_XMMS xmm2, r11, r9, r10, .Lxmm_setup_finished_\suffix 1234 LOOP_OVER_SHORTY_LOADING_XMMS xmm3, r11, r9, r10, .Lxmm_setup_finished_\suffix 1235 LOOP_OVER_SHORTY_LOADING_XMMS xmm4, r11, r9, r10, .Lxmm_setup_finished_\suffix 1236.Lxmm_setup_finished_\suffix: 1237 leaq 1(rINSTq), %r9 // shorty + 1 ; ie skip return arg character 1238 movzwl 4(rPC), %r11d // arguments 1239 .if \is_string_init 1240 movq $$1, %r10 // ignore first argument 1241 shrq MACRO_LITERAL(4), %r11 1242 LOOP_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r9, r10, .Lgpr_setup_finished_\suffix 1243 .elseif \is_static 1244 movq $$0, %r10 // arg_index 1245 LOOP_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r9, r10, .Lgpr_setup_finished_\suffix 1246 .else 1247 shrq MACRO_LITERAL(4), %r11 1248 movq $$1, %r10 // arg_index 1249 .endif 1250 LOOP_OVER_SHORTY_LOADING_GPRS rdx, edx, r11, r9, r10, .Lgpr_setup_finished_\suffix 1251 LOOP_OVER_SHORTY_LOADING_GPRS rcx, ecx, r11, r9, r10, .Lgpr_setup_finished_\suffix 1252 LOOP_OVER_SHORTY_LOADING_GPRS r8, r8d, r11, r9, r10, .Lgpr_setup_finished_\suffix 1253 LOOP_OVER_SHORTY_LOADING_GPRS r9, r9d, r11, r9, r10, .Lgpr_setup_finished_\suffix 1254.Lgpr_setup_finished_\suffix: 1255 .if \is_polymorphic 1256 call SYMBOL(art_quick_invoke_polymorphic) 1257 .elseif \is_custom 1258 call SYMBOL(art_quick_invoke_custom) 1259 .else 1260 .if \is_interface 1261 movq %xmm12, %rax 1262 .endif 1263 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 1264 .endif 1265 cmpb LITERAL(68), (rINSTq) // Test if result type char == 'D'. 1266 je .Lreturn_double_\suffix 1267 cmpb LITERAL(70), (rINSTq) // Test if result type char == 'F'. 1268 jne .Ldone_return_\suffix 1269.Lreturn_float_\suffix: 1270 movd %xmm0, %eax 1271 jmp .Ldone_return_\suffix 1272.Lreturn_double_\suffix: 1273 movq %xmm0, %rax 1274.Ldone_return_\suffix: 1275 /* resume execution of caller */ 1276 .if \is_string_init 1277 movzwl 4(rPC), %r11d // arguments 1278 andq $$0xf, %r11 1279 GET_VREG %esi, %r11 1280 UPDATE_REGISTERS_FOR_STRING_INIT %esi, %eax 1281 .endif 1282 1283 .if \is_polymorphic 1284 ADVANCE_PC_FETCH_AND_GOTO_NEXT 4 1285 .else 1286 ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 1287 .endif 1288.endm 1289 1290.macro COMMON_INVOKE_RANGE is_static=0, is_interface=0, suffix="", is_string_init=0, is_polymorphic=0, is_custom=0 1291 .if \is_polymorphic 1292 // We always go to compiled code for polymorphic calls. 1293 .elseif \is_custom 1294 // We always go to compiled code for custom calls. 1295 .else 1296 DO_ENTRY_POINT_CHECK .Lcall_compiled_code_range_\suffix 1297 .if \is_string_init 1298 call nterp_to_nterp_string_init_range 1299 .elseif \is_static 1300 call nterp_to_nterp_static_range 1301 .else 1302 call nterp_to_nterp_instance_range 1303 .endif 1304 jmp .Ldone_return_range_\suffix 1305 .endif 1306 1307.Lcall_compiled_code_range_\suffix: 1308 .if \is_polymorphic 1309 // No fast path for polymorphic calls. 1310 .elseif \is_custom 1311 // No fast path for custom calls. 1312 .elseif \is_string_init 1313 // No fast path for string.init. 1314 .else 1315 testl $$ART_METHOD_NTERP_INVOKE_FAST_PATH_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi) 1316 je .Lfast_path_with_few_args_range_\suffix 1317 movzbl 1(rPC), %r9d // number of arguments 1318 .if \is_static 1319 testl %r9d, %r9d 1320 je .Linvoke_fast_path_range_\suffix 1321 .else 1322 cmpl MACRO_LITERAL(1), %r9d 1323 je .Linvoke_fast_path_range_\suffix 1324 .endif 1325 movzwl 4(rPC), %r11d // dex register of first argument 1326 leaq (rFP, %r11, 4), %r11 // location of first dex register value 1327 cmpl MACRO_LITERAL(2), %r9d 1328 .if \is_static 1329 jl .Lone_arg_fast_path_range_\suffix 1330 .endif 1331 je .Ltwo_args_fast_path_range_\suffix 1332 cmp MACRO_LITERAL(4), %r9d 1333 jl .Lthree_args_fast_path_range_\suffix 1334 je .Lfour_args_fast_path_range_\suffix 1335 cmp MACRO_LITERAL(5), %r9d 1336 je .Lfive_args_fast_path_range_\suffix 1337 1338.Lloop_over_fast_path_range_\suffix: 1339 subl MACRO_LITERAL(1), %r9d 1340 movl (%r11, %r9, 4), %r8d 1341 movl %r8d, 8(%rsp, %r9, 4) // Add 8 for the ArtMethod 1342 cmpl MACRO_LITERAL(5), %r9d 1343 jne .Lloop_over_fast_path_range_\suffix 1344 1345.Lfive_args_fast_path_range_\suffix: 1346 movl 16(%r11), %r9d 1347.Lfour_args_fast_path_range_\suffix: 1348 movl 12(%r11), %r8d 1349.Lthree_args_fast_path_range_\suffix: 1350 movl 8(%r11), %ecx 1351.Ltwo_args_fast_path_range_\suffix: 1352 movl 4(%r11), %edx 1353.Lone_arg_fast_path_range_\suffix: 1354 .if \is_static 1355 movl 0(%r11), %esi 1356 .else 1357 // First argument already in %esi. 1358 .endif 1359.Linvoke_fast_path_range_\suffix: 1360 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 1361 ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 1362 1363.Lfast_path_with_few_args_range_\suffix: 1364 // Fast path when we have zero or one argument (modulo 'this'). If there 1365 // is one argument, we can put it in both floating point and core register. 1366 movzbl 1(rPC), %r9d # Number of arguments 1367 .if \is_static 1368 cmpl MACRO_LITERAL(1), %r9d 1369 jl .Linvoke_with_few_args_range_\suffix 1370 jne .Lget_shorty_range_\suffix 1371 movzwl 4(rPC), %r9d // Dex register of first argument 1372 GET_VREG %esi, %r9 1373 movd %esi, %xmm0 1374 .else 1375 cmpl MACRO_LITERAL(2), %r9d 1376 jl .Linvoke_with_few_args_range_\suffix 1377 jne .Lget_shorty_range_\suffix 1378 movzwl 4(rPC), %r9d 1379 addl MACRO_LITERAL(1), %r9d // dex register of second argument 1380 GET_VREG %edx, %r9 1381 movd %edx, %xmm0 1382 .endif 1383.Linvoke_with_few_args_range_\suffix: 1384 // Check if the next instruction is move-result or move-result-wide. 1385 // If it is, we fetch the shorty and jump to the regular invocation. 1386 movzwq 6(rPC), %r9 1387 and MACRO_LITERAL(0xfe), %r9d 1388 cmpl MACRO_LITERAL(0x0a), %r9d 1389 je .Lget_shorty_and_invoke_range_\suffix 1390 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 1391 ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 1392.Lget_shorty_and_invoke_range_\suffix: 1393 .if \is_interface 1394 // Save interface method, used for conflict resolution, in a callee-save register. 1395 movq %rax, %xmm12 1396 .endif 1397 GET_SHORTY_SLOW_PATH rINSTq, \is_interface 1398 jmp .Lgpr_setup_finished_range_\suffix 1399 .endif 1400 1401.Lget_shorty_range_\suffix: 1402 .if \is_interface 1403 // Save interface method, used for conflict resolution, in a callee-saved register. 1404 movq %rax, %xmm12 1405 .endif 1406 GET_SHORTY rINSTq, \is_interface, \is_polymorphic, \is_custom 1407 // From this point: 1408 // - rINSTq contains shorty (in callee-save to switch over return value after call). 1409 // - rdi contains method 1410 // - rsi contains 'this' pointer for instance method. 1411 leaq 1(rINSTq), %r9 // shorty + 1 ; ie skip return arg character 1412 movzwl 4(rPC), %r10d // arg start index 1413 .if \is_string_init 1414 addq $$1, %r10 // arg start index 1415 movq $$1, %rbp // index in stack 1416 .elseif \is_static 1417 movq $$0, %rbp // index in stack 1418 .else 1419 addq $$1, %r10 // arg start index 1420 movq $$1, %rbp // index in stack 1421 .endif 1422 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm0, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1423 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm1, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1424 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm2, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1425 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm3, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1426 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm4, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1427 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm5, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1428 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm6, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1429 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm7, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1430 LOOP_RANGE_OVER_FPs r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1431.Lxmm_setup_finished_range_\suffix: 1432 leaq 1(%rbx), %r11 // shorty + 1 ; ie skip return arg character 1433 movzwl 4(rPC), %r10d // arg start index 1434 .if \is_string_init 1435 addq $$1, %r10 // arg start index 1436 movq $$1, %rbp // index in stack 1437 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1438 .elseif \is_static 1439 movq $$0, %rbp // index in stack 1440 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1441 .else 1442 addq $$1, %r10 // arg start index 1443 movq $$1, %rbp // index in stack 1444 .endif 1445 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rdx, edx, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1446 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rcx, ecx, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1447 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS r8, r8d, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1448 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS r9, r9d, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1449 LOOP_RANGE_OVER_INTs r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1450 1451.Lgpr_setup_finished_range_\suffix: 1452 .if \is_polymorphic 1453 call SYMBOL(art_quick_invoke_polymorphic) 1454 .elseif \is_custom 1455 call SYMBOL(art_quick_invoke_custom) 1456 .else 1457 .if \is_interface 1458 // Set the hidden argument for conflict resolution. 1459 movq %xmm12, %rax 1460 .endif 1461 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 1462 .endif 1463 cmpb LITERAL(68), (%rbx) // Test if result type char == 'D'. 1464 je .Lreturn_range_double_\suffix 1465 cmpb LITERAL(70), (%rbx) // Test if result type char == 'F'. 1466 je .Lreturn_range_float_\suffix 1467 /* resume execution of caller */ 1468.Ldone_return_range_\suffix: 1469 .if \is_string_init 1470 movzwl 4(rPC), %r11d // arguments 1471 GET_VREG %esi, %r11 1472 UPDATE_REGISTERS_FOR_STRING_INIT %esi, %eax 1473 .endif 1474 1475 .if \is_polymorphic 1476 ADVANCE_PC_FETCH_AND_GOTO_NEXT 4 1477 .else 1478 ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 1479 .endif 1480.Lreturn_range_double_\suffix: 1481 movq %xmm0, %rax 1482 jmp .Ldone_return_range_\suffix 1483.Lreturn_range_float_\suffix: 1484 movd %xmm0, %eax 1485 jmp .Ldone_return_range_\suffix 1486.endm 1487 1488// Fetch some information from the thread cache. 1489// Uses rax, rdx, rcx as temporaries. 1490.macro FETCH_FROM_THREAD_CACHE dest_reg, slow_path 1491 movq rSELF:THREAD_SELF_OFFSET, %rax 1492 movq rPC, %rdx 1493 salq MACRO_LITERAL(THREAD_INTERPRETER_CACHE_SIZE_SHIFT), %rdx 1494 andq MACRO_LITERAL(THREAD_INTERPRETER_CACHE_SIZE_MASK), %rdx 1495 cmpq THREAD_INTERPRETER_CACHE_OFFSET(%rax, %rdx, 1), rPC 1496 jne \slow_path 1497 movq __SIZEOF_POINTER__+THREAD_INTERPRETER_CACHE_OFFSET(%rax, %rdx, 1), \dest_reg 1498.endm 1499 1500// Helper for static field get. 1501.macro OP_SGET load="movl", wide="0" 1502 // Fast-path which gets the field from thread-local cache. 1503 FETCH_FROM_THREAD_CACHE %rax, 2f 15041: 1505 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 1506 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 1507 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1508 jne 3f 15094: 1510 .if \wide 1511 movq (%eax,%edx,1), %rax 1512 SET_WIDE_VREG %rax, rINSTq # fp[A] <- value 1513 .else 1514 \load (%eax, %edx, 1), %eax 1515 SET_VREG %eax, rINSTq # fp[A] <- value 1516 .endif 1517 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 15182: 1519 EXPORT_PC 1520 movq rSELF:THREAD_SELF_OFFSET, %rdi 1521 movq 0(%rsp), %rsi 1522 movq rPC, %rdx 1523 movq $$0, %rcx 1524 call nterp_get_static_field 1525 // Clear the marker that we put for volatile fields. The x86 memory 1526 // model doesn't require a barrier. 1527 andq $$-2, %rax 1528 jmp 1b 15293: 1530 call art_quick_read_barrier_mark_reg00 1531 jmp 4b 1532.endm 1533 1534// Helper for static field put. 1535.macro OP_SPUT rINST_reg="rINST", store="movl", wide="0": 1536 // Fast-path which gets the field from thread-local cache. 1537 FETCH_FROM_THREAD_CACHE %rax, 2f 15381: 1539 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 1540 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 1541 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1542 jne 3f 15434: 1544 .if \wide 1545 GET_WIDE_VREG rINSTq, rINSTq # rINST <- v[A] 1546 .else 1547 GET_VREG rINST, rINSTq # rINST <- v[A] 1548 .endif 1549 \store \rINST_reg, (%rax,%rdx,1) 1550 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 15512: 1552 EXPORT_PC 1553 movq rSELF:THREAD_SELF_OFFSET, %rdi 1554 movq 0(%rsp), %rsi 1555 movq rPC, %rdx 1556 movq $$0, %rcx 1557 call nterp_get_static_field 1558 testq MACRO_LITERAL(1), %rax 1559 je 1b 1560 // Clear the marker that we put for volatile fields. The x86 memory 1561 // model doesn't require a barrier. 1562 CLEAR_VOLATILE_MARKER %rax 1563 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 1564 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 1565 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1566 jne 6f 15675: 1568 .if \wide 1569 GET_WIDE_VREG rINSTq, rINSTq # rINST <- v[A] 1570 .else 1571 GET_VREG rINST, rINSTq # rINST <- v[A] 1572 .endif 1573 \store \rINST_reg, (%rax,%rdx,1) 1574 lock addl $$0, (%rsp) 1575 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 15763: 1577 call art_quick_read_barrier_mark_reg00 1578 jmp 4b 15796: 1580 call art_quick_read_barrier_mark_reg00 1581 jmp 5b 1582.endm 1583 1584 1585.macro OP_IPUT_INTERNAL rINST_reg="rINST", store="movl", wide="0": 1586 movzbq rINSTbl, %rcx # rcx <- BA 1587 sarl $$4, %ecx # ecx <- B 1588 GET_VREG %ecx, %rcx # vB (object we're operating on) 1589 testl %ecx, %ecx # is object null? 1590 je common_errNullObject 1591 andb $$0xf, rINSTbl # rINST <- A 1592 .if \wide 1593 GET_WIDE_VREG rINSTq, rINSTq # rax<- fp[A]/fp[A+1] 1594 .else 1595 GET_VREG rINST, rINSTq # rINST <- v[A] 1596 .endif 1597 \store \rINST_reg, (%rcx,%rax,1) 1598.endm 1599 1600// Helper for instance field put. 1601.macro OP_IPUT rINST_reg="rINST", store="movl", wide="0": 1602 // Fast-path which gets the field from thread-local cache. 1603 FETCH_FROM_THREAD_CACHE %rax, 2f 16041: 1605 OP_IPUT_INTERNAL \rINST_reg, \store, \wide 1606 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 16072: 1608 EXPORT_PC 1609 movq rSELF:THREAD_SELF_OFFSET, %rdi 1610 movq 0(%rsp), %rsi 1611 movq rPC, %rdx 1612 movq $$0, %rcx 1613 call nterp_get_instance_field_offset 1614 testl %eax, %eax 1615 jns 1b 1616 negl %eax 1617 OP_IPUT_INTERNAL \rINST_reg, \store, \wide 1618 lock addl $$0, (%rsp) 1619 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 1620.endm 1621 1622// Helper for instance field get. 1623.macro OP_IGET load="movl", wide="0" 1624 // Fast-path which gets the field from thread-local cache. 1625 FETCH_FROM_THREAD_CACHE %rax, 2f 16261: 1627 movl rINST, %ecx # rcx <- BA 1628 sarl $$4, %ecx # ecx <- B 1629 GET_VREG %ecx, %rcx # vB (object we're operating on) 1630 testl %ecx, %ecx # is object null? 1631 je common_errNullObject 1632 andb $$0xf,rINSTbl # rINST <- A 1633 .if \wide 1634 movq (%rcx,%rax,1), %rax 1635 SET_WIDE_VREG %rax, rINSTq # fp[A] <- value 1636 .else 1637 \load (%rcx,%rax,1), %eax 1638 SET_VREG %eax, rINSTq # fp[A] <- value 1639 .endif 1640 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 16412: 1642 EXPORT_PC 1643 movq rSELF:THREAD_SELF_OFFSET, %rdi 1644 movq 0(%rsp), %rsi 1645 movq rPC, %rdx 1646 movq $$0, %rcx 1647 call nterp_get_instance_field_offset 1648 testl %eax, %eax 1649 jns 1b 1650 negl %eax 1651 jmp 1b 1652.endm 1653 1654.macro SETUP_REFERENCE_PARAMETER_IN_GPR gpr32, regs, refs, ins, arg_offset, finished 1655 movl REG_VAR(gpr32), (REG_VAR(regs), REG_VAR(arg_offset)) 1656 movl REG_VAR(gpr32), (REG_VAR(refs), REG_VAR(arg_offset)) 1657 addq MACRO_LITERAL(4), REG_VAR(arg_offset) 1658 subl MACRO_LITERAL(1), REG_VAR(ins) 1659 je \finished 1660.endm 1661 1662// Uses eax as temporary 1663.macro SETUP_REFERENCE_PARAMETERS_IN_STACK regs, refs, ins, stack_ptr, arg_offset 16641: 1665 movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_offset)), %eax 1666 movl %eax, (REG_VAR(regs), REG_VAR(arg_offset)) 1667 movl %eax, (REG_VAR(refs), REG_VAR(arg_offset)) 1668 addq MACRO_LITERAL(4), REG_VAR(arg_offset) 1669 subl MACRO_LITERAL(1), REG_VAR(ins) 1670 jne 1b 1671.endm 1672 1673%def entry(): 1674/* 1675 * ArtMethod entry point. 1676 * 1677 * On entry: 1678 * rdi ArtMethod* callee 1679 * rest method parameters 1680 */ 1681 1682OAT_ENTRY ExecuteNterpImpl, EndExecuteNterpImpl 1683 .cfi_startproc 1684 .cfi_def_cfa rsp, 8 1685 testq %rax, -STACK_OVERFLOW_RESERVED_BYTES(%rsp) 1686 /* Spill callee save regs */ 1687 SPILL_ALL_CALLEE_SAVES 1688 1689 movq ART_METHOD_DATA_OFFSET_64(%rdi), rPC 1690 1691 // Setup the stack for executing the method. 1692 SETUP_STACK_FRAME rPC, rREFS, rREFS32, rFP, CFI_REFS, load_ins=1 1693 1694 // Setup the parameters 1695 testl %r14d, %r14d 1696 je .Lxmm_setup_finished 1697 1698 subq %r14, %rbx 1699 salq $$2, %rbx // rbx is now the offset for inputs into the registers array. 1700 1701 testl $$ART_METHOD_NTERP_ENTRY_POINT_FAST_PATH_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi) 1702 je .Lsetup_slow_path 1703 leaq (rFP, %rbx, 1), %rdi 1704 leaq (rREFS, %rbx, 1), %rbx 1705 movq $$0, %r10 1706 1707 SETUP_REFERENCE_PARAMETER_IN_GPR esi, rdi, rbx, r14d, r10, .Lxmm_setup_finished 1708 SETUP_REFERENCE_PARAMETER_IN_GPR edx, rdi, rbx, r14d, r10, .Lxmm_setup_finished 1709 SETUP_REFERENCE_PARAMETER_IN_GPR ecx, rdi, rbx, r14d, r10, .Lxmm_setup_finished 1710 SETUP_REFERENCE_PARAMETER_IN_GPR r8d, rdi, rbx, r14d, r10, .Lxmm_setup_finished 1711 SETUP_REFERENCE_PARAMETER_IN_GPR r9d, rdi, rbx, r14d, r10, .Lxmm_setup_finished 1712 SETUP_REFERENCE_PARAMETERS_IN_STACK rdi, rbx, r14d, r11, r10 1713 jmp .Lxmm_setup_finished 1714 1715.Lsetup_slow_path: 1716 // If the method is not static and there is one argument ('this'), we don't need to fetch the 1717 // shorty. 1718 testl $$ART_METHOD_IS_STATIC_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi) 1719 jne .Lsetup_with_shorty 1720 1721 movl %esi, (rFP, %rbx) 1722 movl %esi, (rREFS, %rbx) 1723 1724 cmpl $$1, %r14d 1725 je .Lxmm_setup_finished 1726 1727.Lsetup_with_shorty: 1728 // TODO: Get shorty in a better way and remove below 1729 push %rdi 1730 push %rsi 1731 push %rdx 1732 push %rcx 1733 push %r8 1734 push %r9 1735 1736 // Save xmm registers + alignment. 1737 subq MACRO_LITERAL(8 * 8 + 8), %rsp 1738 movq %xmm0, 0(%rsp) 1739 movq %xmm1, 8(%rsp) 1740 movq %xmm2, 16(%rsp) 1741 movq %xmm3, 24(%rsp) 1742 movq %xmm4, 32(%rsp) 1743 movq %xmm5, 40(%rsp) 1744 movq %xmm6, 48(%rsp) 1745 movq %xmm7, 56(%rsp) 1746 1747 call SYMBOL(NterpGetShorty) 1748 // Save shorty in callee-save rbp. 1749 movq %rax, %rbp 1750 1751 // Restore xmm registers + alignment. 1752 movq 0(%rsp), %xmm0 1753 movq 8(%rsp), %xmm1 1754 movq 16(%rsp), %xmm2 1755 movq 24(%rsp), %xmm3 1756 movq 32(%rsp), %xmm4 1757 movq 40(%rsp), %xmm5 1758 movq 48(%rsp), %xmm6 1759 movq 56(%rsp), %xmm7 1760 addq MACRO_LITERAL(8 * 8 + 8), %rsp 1761 1762 pop %r9 1763 pop %r8 1764 pop %rcx 1765 pop %rdx 1766 pop %rsi 1767 pop %rdi 1768 // Reload the old stack pointer, which used to be stored in %r11, which is not callee-saved. 1769 movq -8(rREFS), %r11 1770 // TODO: Get shorty in a better way and remove above 1771 1772 movq $$0, %r14 1773 testl $$ART_METHOD_IS_STATIC_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi) 1774 1775 // Available: rdi, r10 1776 // Note the leaq below don't change the flags. 1777 leaq 1(%rbp), %r10 // shorty + 1 ; ie skip return arg character 1778 leaq (rFP, %rbx, 1), %rdi 1779 leaq (rREFS, %rbx, 1), %rbx 1780 jne .Lhandle_static_method 1781 addq $$4, %rdi 1782 addq $$4, %rbx 1783 addq $$4, %r11 1784 jmp .Lcontinue_setup_gprs 1785.Lhandle_static_method: 1786 LOOP_OVER_SHORTY_STORING_GPRS rsi, esi, r10, r14, rdi, rbx, .Lgpr_setup_finished 1787.Lcontinue_setup_gprs: 1788 LOOP_OVER_SHORTY_STORING_GPRS rdx, edx, r10, r14, rdi, rbx, .Lgpr_setup_finished 1789 LOOP_OVER_SHORTY_STORING_GPRS rcx, ecx, r10, r14, rdi, rbx, .Lgpr_setup_finished 1790 LOOP_OVER_SHORTY_STORING_GPRS r8, r8d, r10, r14, rdi, rbx, .Lgpr_setup_finished 1791 LOOP_OVER_SHORTY_STORING_GPRS r9, r9d, r10, r14, rdi, rbx, .Lgpr_setup_finished 1792 LOOP_OVER_INTs r10, r14, rdi, rbx, r11, .Lgpr_setup_finished 1793.Lgpr_setup_finished: 1794 leaq 1(%rbp), %r10 // shorty + 1 ; ie skip return arg character 1795 movq $$0, %r14 // reset counter 1796 LOOP_OVER_SHORTY_STORING_XMMS xmm0, r10, r14, rdi, .Lxmm_setup_finished 1797 LOOP_OVER_SHORTY_STORING_XMMS xmm1, r10, r14, rdi, .Lxmm_setup_finished 1798 LOOP_OVER_SHORTY_STORING_XMMS xmm2, r10, r14, rdi, .Lxmm_setup_finished 1799 LOOP_OVER_SHORTY_STORING_XMMS xmm3, r10, r14, rdi, .Lxmm_setup_finished 1800 LOOP_OVER_SHORTY_STORING_XMMS xmm4, r10, r14, rdi, .Lxmm_setup_finished 1801 LOOP_OVER_SHORTY_STORING_XMMS xmm5, r10, r14, rdi, .Lxmm_setup_finished 1802 LOOP_OVER_SHORTY_STORING_XMMS xmm6, r10, r14, rdi, .Lxmm_setup_finished 1803 LOOP_OVER_SHORTY_STORING_XMMS xmm7, r10, r14, rdi, .Lxmm_setup_finished 1804 LOOP_OVER_FPs r10, r14, rdi, r11, .Lxmm_setup_finished 1805.Lxmm_setup_finished: 1806 CFI_DEFINE_DEX_PC_WITH_OFFSET(CFI_TMP, CFI_DEX, 0) 1807 1808 // Set rIBASE 1809 leaq artNterpAsmInstructionStart(%rip), rIBASE 1810 /* start executing the instruction at rPC */ 1811 START_EXECUTING_INSTRUCTIONS 1812 /* NOTE: no fallthrough */ 1813 // cfi info continues, and covers the whole nterp implementation. 1814 END ExecuteNterpImpl 1815 1816%def opcode_pre(): 1817 1818%def helpers(): 1819 1820%def footer(): 1821/* 1822 * =========================================================================== 1823 * Common subroutines and data 1824 * =========================================================================== 1825 */ 1826 1827 .text 1828 .align 2 1829 1830// Enclose all code below in a symbol (which gets printed in backtraces). 1831ENTRY nterp_helper 1832 1833// Note: mterp also uses the common_* names below for helpers, but that's OK 1834// as the C compiler compiled each interpreter separately. 1835common_errDivideByZero: 1836 EXPORT_PC 1837 call art_quick_throw_div_zero 1838 1839// Expect array in edi, index in esi. 1840common_errArrayIndex: 1841 EXPORT_PC 1842 movl MIRROR_ARRAY_LENGTH_OFFSET(%edi), %eax 1843 movl %esi, %edi 1844 movl %eax, %esi 1845 call art_quick_throw_array_bounds 1846 1847common_errNullObject: 1848 EXPORT_PC 1849 call art_quick_throw_null_pointer_exception 1850 1851NterpCommonInvokeStatic: 1852 COMMON_INVOKE_NON_RANGE is_static=1, is_interface=0, suffix="invokeStatic" 1853 1854NterpCommonInvokeStaticRange: 1855 COMMON_INVOKE_RANGE is_static=1, is_interface=0, suffix="invokeStatic" 1856 1857NterpCommonInvokeInstance: 1858 COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, suffix="invokeInstance" 1859 1860NterpCommonInvokeInstanceRange: 1861 COMMON_INVOKE_RANGE is_static=0, is_interface=0, suffix="invokeInstance" 1862 1863NterpCommonInvokeInterface: 1864 COMMON_INVOKE_NON_RANGE is_static=0, is_interface=1, suffix="invokeInterface" 1865 1866NterpCommonInvokeInterfaceRange: 1867 COMMON_INVOKE_RANGE is_static=0, is_interface=1, suffix="invokeInterface" 1868 1869NterpCommonInvokePolymorphic: 1870 COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, is_string_init=0, is_polymorphic=1, suffix="invokePolymorphic" 1871 1872NterpCommonInvokePolymorphicRange: 1873 COMMON_INVOKE_RANGE is_static=0, is_interface=0, is_polymorphic=1, suffix="invokePolymorphic" 1874 1875NterpCommonInvokeCustom: 1876 COMMON_INVOKE_NON_RANGE is_static=1, is_interface=0, is_string_init=0, is_polymorphic=0, is_custom=1, suffix="invokeCustom" 1877 1878NterpCommonInvokeCustomRange: 1879 COMMON_INVOKE_RANGE is_static=1, is_interface=0, is_polymorphic=0, is_custom=1, suffix="invokeCustom" 1880 1881NterpHandleStringInit: 1882 COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, is_string_init=1, suffix="stringInit" 1883 1884NterpHandleStringInitRange: 1885 COMMON_INVOKE_RANGE is_static=0, is_interface=0, is_string_init=1, suffix="stringInit" 1886 1887NterpNewInstance: 1888 EXPORT_PC 1889 // Fast-path which gets the class from thread-local cache. 1890 FETCH_FROM_THREAD_CACHE %rdi, 2f 1891 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1892 jne 3f 18934: 1894 callq *rSELF:THREAD_ALLOC_OBJECT_ENTRYPOINT_OFFSET 18951: 1896 SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value 1897 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 18982: 1899 movq rSELF:THREAD_SELF_OFFSET, %rdi 1900 movq 0(%rsp), %rsi 1901 movq rPC, %rdx 1902 call nterp_get_class_or_allocate_object 1903 jmp 1b 19043: 1905 // 07 is %rdi 1906 call art_quick_read_barrier_mark_reg07 1907 jmp 4b 1908 1909NterpNewArray: 1910 /* new-array vA, vB, class@CCCC */ 1911 EXPORT_PC 1912 // Fast-path which gets the class from thread-local cache. 1913 FETCH_FROM_THREAD_CACHE %rdi, 2f 1914 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1915 jne 3f 19161: 1917 movzbl rINSTbl,%esi 1918 sarl $$4,%esi # esi<- B 1919 GET_VREG %esi %rsi # esi<- vB (array length) 1920 andb $$0xf,rINSTbl # rINST<- A 1921 callq *rSELF:THREAD_ALLOC_ARRAY_ENTRYPOINT_OFFSET 1922 SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value 1923 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 19242: 1925 movq rSELF:THREAD_SELF_OFFSET, %rdi 1926 movq 0(%rsp), %rsi 1927 movq rPC, %rdx 1928 call nterp_get_class_or_allocate_object 1929 movq %rax, %rdi 1930 jmp 1b 19313: 1932 // 07 is %rdi 1933 call art_quick_read_barrier_mark_reg07 1934 jmp 1b 1935 1936NterpPutObjectInstanceField: 1937 movl rINST, %ebp # rbp <- BA 1938 andl $$0xf, %ebp # rbp <- A 1939 GET_VREG %ecx, %rbp # ecx <- v[A] 1940 sarl $$4, rINST 1941 // Fast-path which gets the field from thread-local cache. 1942 FETCH_FROM_THREAD_CACHE %rax, 2f 19431: 1944 GET_VREG rINST, rINSTq # vB (object we're operating on) 1945 testl rINST, rINST # is object null? 1946 je common_errNullObject 1947 movl %ecx, (rINSTq,%rax,1) 1948 testl %ecx, %ecx 1949 je 4f 1950 movq rSELF:THREAD_CARD_TABLE_OFFSET, %rax 1951 shrq $$CARD_TABLE_CARD_SHIFT, rINSTq 1952 movb %al, (%rax, rINSTq, 1) 19534: 1954 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 19552: 1956 EXPORT_PC 1957 movq rSELF:THREAD_SELF_OFFSET, %rdi 1958 movq 0(%rsp), %rsi 1959 movq rPC, %rdx 1960 // %rcx is already set. 1961 call nterp_get_instance_field_offset 1962 // Reload the value as it may have moved. 1963 GET_VREG %ecx, %rbp # ecx <- v[A] 1964 testl %eax, %eax 1965 jns 1b 1966 GET_VREG rINST, rINSTq # vB (object we're operating on) 1967 testl rINST, rINST # is object null? 1968 je common_errNullObject 1969 negl %eax 1970 movl %ecx, (rINSTq,%rax,1) 1971 testl %ecx, %ecx 1972 je 5f 1973 movq rSELF:THREAD_CARD_TABLE_OFFSET, %rax 1974 shrq $$CARD_TABLE_CARD_SHIFT, rINSTq 1975 movb %al, (%rax, rINSTq, 1) 19765: 1977 lock addl $$0, (%rsp) 1978 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 1979 1980NterpGetObjectInstanceField: 1981 // Fast-path which gets the field from thread-local cache. 1982 FETCH_FROM_THREAD_CACHE %rax, 2f 19831: 1984 movl rINST, %ecx # rcx <- BA 1985 sarl $$4, %ecx # ecx <- B 1986 GET_VREG %ecx, %rcx # vB (object we're operating on) 1987 testl %ecx, %ecx # is object null? 1988 je common_errNullObject 1989 testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%ecx) 1990 movl (%rcx,%rax,1), %eax 1991 jnz 3f 19924: 1993 andb $$0xf,rINSTbl # rINST <- A 1994 SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value 1995 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 19962: 1997 EXPORT_PC 1998 movq rSELF:THREAD_SELF_OFFSET, %rdi 1999 movq 0(%rsp), %rsi 2000 movq rPC, %rdx 2001 movq $$0, %rcx 2002 call nterp_get_instance_field_offset 2003 testl %eax, %eax 2004 jns 1b 2005 // For volatile fields, we return a negative offset. Remove the sign 2006 // and no need for any barrier thanks to the memory model. 2007 negl %eax 2008 jmp 1b 20093: 2010 // reg00 is eax 2011 call art_quick_read_barrier_mark_reg00 2012 jmp 4b 2013 2014NterpPutObjectStaticField: 2015 GET_VREG %ebp, rINSTq 2016 // Fast-path which gets the field from thread-local cache. 2017 FETCH_FROM_THREAD_CACHE %rax, 2f 20181: 2019 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 2020 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 2021 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 2022 jne 3f 20235: 2024 movl %ebp, (%eax, %edx, 1) 2025 testl %ebp, %ebp 2026 je 4f 2027 movq rSELF:THREAD_CARD_TABLE_OFFSET, %rcx 2028 shrq $$CARD_TABLE_CARD_SHIFT, %rax 2029 movb %cl, (%rax, %rcx, 1) 20304: 2031 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 20322: 2033 EXPORT_PC 2034 movq rSELF:THREAD_SELF_OFFSET, %rdi 2035 movq 0(%rsp), %rsi 2036 movq rPC, %rdx 2037 movq %rbp, %rcx 2038 call nterp_get_static_field 2039 // Reload the value as it may have moved. 2040 GET_VREG %ebp, rINSTq 2041 testq MACRO_LITERAL(1), %rax 2042 je 1b 2043 CLEAR_VOLATILE_MARKER %rax 2044 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 2045 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 2046 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 2047 jne 7f 20486: 2049 movl %ebp, (%eax, %edx, 1) 2050 testl %ebp, %ebp 2051 je 8f 2052 movq rSELF:THREAD_CARD_TABLE_OFFSET, %rcx 2053 shrq $$CARD_TABLE_CARD_SHIFT, %rax 2054 movb %cl, (%rax, %rcx, 1) 20558: 2056 lock addl $$0, (%rsp) 2057 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 20583: 2059 call art_quick_read_barrier_mark_reg00 2060 jmp 5b 20617: 2062 call art_quick_read_barrier_mark_reg00 2063 jmp 6b 2064 2065NterpGetObjectStaticField: 2066 // Fast-path which gets the field from thread-local cache. 2067 FETCH_FROM_THREAD_CACHE %rax, 2f 20681: 2069 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 2070 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 2071 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 2072 jne 5f 20736: 2074 testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%eax) 2075 movl (%eax, %edx, 1), %eax 2076 jnz 3f 20774: 2078 SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value 2079 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 20802: 2081 EXPORT_PC 2082 movq rSELF:THREAD_SELF_OFFSET, %rdi 2083 movq 0(%rsp), %rsi 2084 movq rPC, %rdx 2085 movq $$0, %rcx 2086 call nterp_get_static_field 2087 andq $$-2, %rax 2088 jmp 1b 20893: 2090 call art_quick_read_barrier_mark_reg00 2091 jmp 4b 20925: 2093 call art_quick_read_barrier_mark_reg00 2094 jmp 6b 2095 2096NterpGetBooleanStaticField: 2097 OP_SGET load="movsbl", wide=0 2098 2099NterpGetByteStaticField: 2100 OP_SGET load="movsbl", wide=0 2101 2102NterpGetCharStaticField: 2103 OP_SGET load="movzwl", wide=0 2104 2105NterpGetShortStaticField: 2106 OP_SGET load="movswl", wide=0 2107 2108NterpGetWideStaticField: 2109 OP_SGET load="movq", wide=1 2110 2111NterpGetIntStaticField: 2112 OP_SGET load="movl", wide=0 2113 2114NterpPutStaticField: 2115 OP_SPUT rINST_reg=rINST, store="movl", wide=0 2116 2117NterpPutBooleanStaticField: 2118NterpPutByteStaticField: 2119 OP_SPUT rINST_reg=rINSTbl, store="movb", wide=0 2120 2121NterpPutCharStaticField: 2122NterpPutShortStaticField: 2123 OP_SPUT rINST_reg=rINSTw, store="movw", wide=0 2124 2125NterpPutWideStaticField: 2126 OP_SPUT rINST_reg=rINSTq, store="movq", wide=1 2127 2128NterpPutInstanceField: 2129 OP_IPUT rINST_reg=rINST, store="movl", wide=0 2130 2131NterpPutBooleanInstanceField: 2132NterpPutByteInstanceField: 2133 OP_IPUT rINST_reg=rINSTbl, store="movb", wide=0 2134 2135NterpPutCharInstanceField: 2136NterpPutShortInstanceField: 2137 OP_IPUT rINST_reg=rINSTw, store="movw", wide=0 2138 2139NterpPutWideInstanceField: 2140 OP_IPUT rINST_reg=rINSTq, store="movq", wide=1 2141 2142NterpGetBooleanInstanceField: 2143 OP_IGET load="movzbl", wide=0 2144 2145NterpGetByteInstanceField: 2146 OP_IGET load="movsbl", wide=0 2147 2148NterpGetCharInstanceField: 2149 OP_IGET load="movzwl", wide=0 2150 2151NterpGetShortInstanceField: 2152 OP_IGET load="movswl", wide=0 2153 2154NterpGetWideInstanceField: 2155 OP_IGET load="movq", wide=1 2156 2157NterpGetInstanceField: 2158 OP_IGET load="movl", wide=0 2159 2160NterpInstanceOf: 2161 /* instance-of vA, vB, class@CCCC */ 2162 // Fast-path which gets the class from thread-local cache. 2163 EXPORT_PC 2164 FETCH_FROM_THREAD_CACHE %rsi, 2f 2165 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 2166 jne 5f 21671: 2168 movzbl rINSTbl,%edi 2169 sarl $$4,%edi # edi<- B 2170 GET_VREG %edi %rdi # edi<- vB (object) 2171 andb $$0xf,rINSTbl # rINST<- A 2172 testl %edi, %edi 2173 je 3f 2174 call art_quick_instance_of 2175 SET_VREG %eax, rINSTq # fp[A] <- value 21764: 2177 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 21783: 2179 SET_VREG %edi, rINSTq # fp[A] <-0 2180 jmp 4b 21812: 2182 movq rSELF:THREAD_SELF_OFFSET, %rdi 2183 movq 0(%rsp), %rsi 2184 movq rPC, %rdx 2185 call nterp_get_class_or_allocate_object 2186 movq %rax, %rsi 2187 jmp 1b 21885: 2189 // 06 is %rsi 2190 call art_quick_read_barrier_mark_reg06 2191 jmp 1b 2192 2193NterpCheckCast: 2194 // Fast-path which gets the class from thread-local cache. 2195 EXPORT_PC 2196 FETCH_FROM_THREAD_CACHE %rsi, 3f 2197 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 2198 jne 4f 21991: 2200 GET_VREG %edi, rINSTq 2201 testl %edi, %edi 2202 je 2f 2203 call art_quick_check_instance_of 22042: 2205 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 22063: 2207 movq rSELF:THREAD_SELF_OFFSET, %rdi 2208 movq 0(%rsp), %rsi 2209 movq rPC, %rdx 2210 call nterp_get_class_or_allocate_object 2211 movq %rax, %rsi 2212 jmp 1b 22134: 2214 // 06 is %rsi 2215 call art_quick_read_barrier_mark_reg06 2216 jmp 1b 2217 2218NterpHandleHotnessOverflow: 2219 leaq (rPC, rINSTq, 2), %rsi 2220 movq rFP, %rdx 2221 call nterp_hot_method 2222 testq %rax, %rax 2223 jne 1f 2224 leaq (rPC, rINSTq, 2), rPC 2225 FETCH_INST 2226 GOTO_NEXT 22271: 2228 // Drop the current frame. 2229 movq -8(rREFS), %rsp 2230 CFI_DEF_CFA(rsp, CALLEE_SAVES_SIZE) 2231 2232 // Setup the new frame 2233 movq OSR_DATA_FRAME_SIZE(%rax), %rcx 2234 // Given stack size contains all callee saved registers, remove them. 2235 subq $$CALLEE_SAVES_SIZE, %rcx 2236 2237 // Remember CFA. 2238 movq %rsp, %rbp 2239 CFI_DEF_CFA_REGISTER(rbp) 2240 2241 subq %rcx, %rsp 2242 movq %rsp, %rdi // rdi := beginning of stack 2243 leaq OSR_DATA_MEMORY(%rax), %rsi // rsi := memory to copy 2244 rep movsb // while (rcx--) { *rdi++ = *rsi++ } 2245 2246 // Fetch the native PC to jump to and save it in a callee-save register. 2247 movq OSR_DATA_NATIVE_PC(%rax), %rbx 2248 2249 // Free the memory holding OSR Data. 2250 movq %rax, %rdi 2251 call free 2252 2253 // Jump to the compiled code. 2254 jmp *%rbx 2255 2256NterpHandleInvokeInterfaceOnObjectMethodRange: 2257 shrl $$16, %eax 2258 movq MIRROR_CLASS_VTABLE_OFFSET_64(%edx, %eax, 8), %rdi 2259 jmp NterpCommonInvokeInstanceRange 2260 2261NterpHandleInvokeInterfaceOnObjectMethod: 2262 shrl $$16, %eax 2263 movq MIRROR_CLASS_VTABLE_OFFSET_64(%edx, %eax, 8), %rdi 2264 jmp NterpCommonInvokeInstance 2265 2266// This is the logical end of ExecuteNterpImpl, where the frame info applies. 2267// EndExecuteNterpImpl includes the methods below as we want the runtime to 2268// see them as part of the Nterp PCs. 2269.cfi_endproc 2270 2271nterp_to_nterp_static_non_range: 2272 .cfi_startproc 2273 .cfi_def_cfa rsp, 8 2274 SETUP_STACK_FOR_INVOKE 2275 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=1, is_string_init=0 2276 .cfi_endproc 2277 2278nterp_to_nterp_string_init_non_range: 2279 .cfi_startproc 2280 .cfi_def_cfa rsp, 8 2281 SETUP_STACK_FOR_INVOKE 2282 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=1 2283 .cfi_endproc 2284 2285nterp_to_nterp_instance_non_range: 2286 .cfi_startproc 2287 .cfi_def_cfa rsp, 8 2288 SETUP_STACK_FOR_INVOKE 2289 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 2290 .cfi_endproc 2291 2292nterp_to_nterp_static_range: 2293 .cfi_startproc 2294 .cfi_def_cfa rsp, 8 2295 SETUP_STACK_FOR_INVOKE 2296 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=1 2297 .cfi_endproc 2298 2299nterp_to_nterp_instance_range: 2300 .cfi_startproc 2301 .cfi_def_cfa rsp, 8 2302 SETUP_STACK_FOR_INVOKE 2303 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0 2304 .cfi_endproc 2305 2306nterp_to_nterp_string_init_range: 2307 .cfi_startproc 2308 .cfi_def_cfa rsp, 8 2309 SETUP_STACK_FOR_INVOKE 2310 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=1 2311 .cfi_endproc 2312 2313END nterp_helper 2314 2315// This is the end of PCs contained by the OatQuickMethodHeader created for the interpreter 2316// entry point. 2317 FUNCTION_TYPE(EndExecuteNterpImpl) 2318 ASM_HIDDEN SYMBOL(EndExecuteNterpImpl) 2319 .global SYMBOL(EndExecuteNterpImpl) 2320SYMBOL(EndExecuteNterpImpl): 2321 2322// Entrypoints into runtime. 2323NTERP_TRAMPOLINE nterp_get_static_field, NterpGetStaticField 2324NTERP_TRAMPOLINE nterp_get_instance_field_offset, NterpGetInstanceFieldOffset 2325NTERP_TRAMPOLINE nterp_filled_new_array, NterpFilledNewArray 2326NTERP_TRAMPOLINE nterp_filled_new_array_range, NterpFilledNewArrayRange 2327NTERP_TRAMPOLINE nterp_get_class_or_allocate_object, NterpGetClassOrAllocateObject 2328NTERP_TRAMPOLINE nterp_get_method, NterpGetMethod 2329NTERP_TRAMPOLINE nterp_hot_method, NterpHotMethod 2330NTERP_TRAMPOLINE nterp_load_object, NterpLoadObject 2331 2332// gen_mterp.py will inline the following definitions 2333// within [ExecuteNterpImpl, EndExecuteNterpImpl). 2334%def instruction_end(): 2335 2336 FUNCTION_TYPE(artNterpAsmInstructionEnd) 2337 ASM_HIDDEN SYMBOL(artNterpAsmInstructionEnd) 2338 .global SYMBOL(artNterpAsmInstructionEnd) 2339SYMBOL(artNterpAsmInstructionEnd): 2340 // artNterpAsmInstructionEnd is used as landing pad for exception handling. 2341 FETCH_INST 2342 GOTO_NEXT 2343 2344%def instruction_start(): 2345 2346 FUNCTION_TYPE(artNterpAsmInstructionStart) 2347 ASM_HIDDEN SYMBOL(artNterpAsmInstructionStart) 2348 .global SYMBOL(artNterpAsmInstructionStart) 2349SYMBOL(artNterpAsmInstructionStart) = .L_op_nop 2350 .text 2351 2352%def default_helper_prefix(): 2353% return "nterp_" 2354 2355%def opcode_start(): 2356 ENTRY nterp_${opcode} 2357%def opcode_end(): 2358 END nterp_${opcode} 2359%def helper_start(name): 2360 ENTRY ${name} 2361%def helper_end(name): 2362 END ${name} 2363