1%def header(): 2/* 3 * Copyright (C) 2019 The Android Open Source Project 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18/* 19 * This is a #include, not a %include, because we want the C pre-processor 20 * to expand the macros into assembler assignment statements. 21 */ 22#include "asm_support.h" 23#include "arch/x86_64/asm_support_x86_64.S" 24 25/** 26 * x86_64 ABI general notes: 27 * 28 * Caller save set: 29 * rax, rdx, rcx, rsi, rdi, r8-r11, st(0)-st(7) 30 * Callee save set: 31 * rbx, rbp, r12-r15 32 * Return regs: 33 * 32-bit in eax 34 * 64-bit in rax 35 * fp on xmm0 36 * 37 * First 8 fp parameters came in xmm0-xmm7. 38 * First 6 non-fp parameters came in rdi, rsi, rdx, rcx, r8, r9. 39 * Other parameters passed on stack, pushed right-to-left. On entry to target, first 40 * param is at 8(%esp). 41 * 42 * Stack must be 16-byte aligned to support SSE in native code. 43 */ 44 45#define IN_ARG3 %rcx 46#define IN_ARG2 %rdx 47#define IN_ARG1 %rsi 48#define IN_ARG0 %rdi 49/* Out Args */ 50#define OUT_ARG3 %rcx 51#define OUT_ARG2 %rdx 52#define OUT_ARG1 %rsi 53#define OUT_ARG0 %rdi 54#define OUT_32_ARG3 %ecx 55#define OUT_32_ARG2 %edx 56#define OUT_32_ARG1 %esi 57#define OUT_32_ARG0 %edi 58#define OUT_FP_ARG1 %xmm1 59#define OUT_FP_ARG0 %xmm0 60 61/* 62 * single-purpose registers, given names for clarity 63 */ 64#define rSELF %gs 65#define rPC %r12 66#define CFI_DEX 12 // DWARF register number of the register holding dex-pc (rPC). 67#define CFI_TMP 5 // DWARF register number of the first argument register (rdi). 68#define rFP %r13 69#define rINST %ebx 70#define rINSTq %rbx 71#define rINSTw %bx 72#define rINSTbh %bh 73#define rINSTbl %bl 74#define rIBASE %r14 75#define rREFS %r15 76#define rREFS32 %r15d 77#define CFI_REFS 15 // DWARF register number of the reference array (r15). 78 79// Temporary registers while setting up a frame. 80#define rNEW_FP %r8 81#define rNEW_REFS %r9 82#define rNEW_REFS32 %r9d 83#define CFI_NEW_REFS 9 84 85/* 86 * Get/set the 32-bit value from a Dalvik register. 87 */ 88#define VREG_ADDRESS(_vreg) (rFP,_vreg,4) 89#define VREG_HIGH_ADDRESS(_vreg) 4(rFP,_vreg,4) 90#define VREG_REF_ADDRESS(_vreg) (rREFS,_vreg,4) 91#define VREG_REF_HIGH_ADDRESS(_vreg) 4(rREFS,_vreg,4) 92 93// Includes the return address implictly pushed on stack by 'call'. 94#define CALLEE_SAVES_SIZE (6 * 8 + 4 * 8 + 1 * 8) 95 96// +8 for the ArtMethod of the caller. 97#define OFFSET_TO_FIRST_ARGUMENT_IN_STACK (CALLEE_SAVES_SIZE + 8) 98 99/* 100 * Refresh rINST. 101 * At enter to handler rINST does not contain the opcode number. 102 * However some utilities require the full value, so this macro 103 * restores the opcode number. 104 */ 105.macro REFRESH_INST _opnum 106 movb rINSTbl, rINSTbh 107 movb $$\_opnum, rINSTbl 108.endm 109 110/* 111 * Fetch the next instruction from rPC into rINSTw. Does not advance rPC. 112 */ 113.macro FETCH_INST 114 movzwq (rPC), rINSTq 115.endm 116 117/* 118 * Remove opcode from rINST, compute the address of handler and jump to it. 119 */ 120.macro GOTO_NEXT 121 movzx rINSTbl,%ecx 122 movzbl rINSTbh,rINST 123 shll MACRO_LITERAL(${handler_size_bits}), %ecx 124 addq rIBASE, %rcx 125 jmp *%rcx 126.endm 127 128/* 129 * Advance rPC by instruction count. 130 */ 131.macro ADVANCE_PC _count 132 leaq 2*\_count(rPC), rPC 133.endm 134 135/* 136 * Advance rPC by instruction count, fetch instruction and jump to handler. 137 */ 138.macro ADVANCE_PC_FETCH_AND_GOTO_NEXT _count 139 ADVANCE_PC \_count 140 FETCH_INST 141 GOTO_NEXT 142.endm 143 144.macro GET_VREG _reg _vreg 145 movl VREG_ADDRESS(\_vreg), \_reg 146.endm 147 148.macro GET_VREG_OBJECT _reg _vreg 149 movl VREG_REF_ADDRESS(\_vreg), \_reg 150.endm 151 152/* Read wide value. */ 153.macro GET_WIDE_VREG _reg _vreg 154 movq VREG_ADDRESS(\_vreg), \_reg 155.endm 156 157.macro SET_VREG _reg _vreg 158 movl \_reg, VREG_ADDRESS(\_vreg) 159 movl MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg) 160.endm 161 162/* Write wide value. reg is clobbered. */ 163.macro SET_WIDE_VREG _reg _vreg 164 movq \_reg, VREG_ADDRESS(\_vreg) 165 xorq \_reg, \_reg 166 movq \_reg, VREG_REF_ADDRESS(\_vreg) 167.endm 168 169.macro SET_VREG_OBJECT _reg _vreg 170 movl \_reg, VREG_ADDRESS(\_vreg) 171 movl \_reg, VREG_REF_ADDRESS(\_vreg) 172.endm 173 174.macro GET_VREG_HIGH _reg _vreg 175 movl VREG_HIGH_ADDRESS(\_vreg), \_reg 176.endm 177 178.macro SET_VREG_HIGH _reg _vreg 179 movl \_reg, VREG_HIGH_ADDRESS(\_vreg) 180 movl MACRO_LITERAL(0), VREG_REF_HIGH_ADDRESS(\_vreg) 181.endm 182 183.macro CLEAR_REF _vreg 184 movl MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg) 185.endm 186 187.macro CLEAR_WIDE_REF _vreg 188 movl MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg) 189 movl MACRO_LITERAL(0), VREG_REF_HIGH_ADDRESS(\_vreg) 190.endm 191 192.macro GET_VREG_XMMs _xmmreg _vreg 193 movss VREG_ADDRESS(\_vreg), \_xmmreg 194.endm 195.macro GET_VREG_XMMd _xmmreg _vreg 196 movsd VREG_ADDRESS(\_vreg), \_xmmreg 197.endm 198.macro SET_VREG_XMMs _xmmreg _vreg 199 movss \_xmmreg, VREG_ADDRESS(\_vreg) 200.endm 201.macro SET_VREG_XMMd _xmmreg _vreg 202 movsd \_xmmreg, VREG_ADDRESS(\_vreg) 203.endm 204 205// An assembly entry for nterp. 206.macro OAT_ENTRY name 207 FUNCTION_TYPE(\name) 208 ASM_HIDDEN SYMBOL(\name) 209 .global SYMBOL(\name) 210 .balign 16 211SYMBOL(\name): 212.endm 213 214.macro ENTRY name 215 .text 216 ASM_HIDDEN SYMBOL(\name) 217 .global SYMBOL(\name) 218 FUNCTION_TYPE(\name) 219SYMBOL(\name): 220.endm 221 222.macro END name 223 SIZE(\name) 224.endm 225 226// Macro for defining entrypoints into runtime. We don't need to save registers 227// (we're not holding references there), but there is no 228// kDontSave runtime method. So just use the kSaveRefsOnly runtime method. 229.macro NTERP_TRAMPOLINE name, helper 230DEFINE_FUNCTION \name 231 SETUP_SAVE_REFS_ONLY_FRAME 232 call \helper 233 RESTORE_SAVE_REFS_ONLY_FRAME 234 cmpq LITERAL(0), %gs:THREAD_EXCEPTION_OFFSET 235 jne nterp_deliver_pending_exception 236 ret 237END_FUNCTION \name 238.endm 239 240.macro CLEAR_VOLATILE_MARKER reg 241 andq MACRO_LITERAL(-2), \reg 242.endm 243 244.macro EXPORT_PC 245 movq rPC, -16(rREFS) 246.endm 247 248 249.macro BRANCH 250 leaq (rPC, rINSTq, 2), rPC 251 // Update method counter and do a suspend check if the branch is negative or zero. 252 testq rINSTq, rINSTq 253 jle 3f 2542: // We use 2 and not 1 for this local label as the users of the BRANCH macro have a 1 label. 255 FETCH_INST 256 GOTO_NEXT 2573: 258 movq (%rsp), %rdi 259 movzwl ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi), %esi 260#if (NTERP_HOTNESS_VALUE != 0) 261#error Expected 0 for hotness value 262#endif 263 // If the counter is at zero, handle this in the runtime. 264 testw %si, %si 265 je NterpHandleHotnessOverflow 266 // Update counter. 267 addl $$-1, %esi 268 movw %si, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) 269 DO_SUSPEND_CHECK continue_label=2b 270 jmp 2b 271.endm 272 273// Expects: 274// - r10, and r11 to be available. 275// Outputs: 276// - \registers contains the dex registers size 277// - \outs contains the outs size 278// - if load_ins is 1, \ins contains the ins 279// - \code_item is replace with a pointer to the instructions 280.macro FETCH_CODE_ITEM_INFO code_item, registers, outs, ins, load_ins 281 // Fetch dex register size. 282 movzwl CODE_ITEM_REGISTERS_SIZE_OFFSET(\code_item), \registers 283 // Fetch outs size. 284 movzwl CODE_ITEM_OUTS_SIZE_OFFSET(\code_item), \outs 285 .if \load_ins 286 movzwl CODE_ITEM_INS_SIZE_OFFSET(\code_item), \ins 287 .endif 288 addq $$CODE_ITEM_INSNS_OFFSET, \code_item 289.endm 290 291// Setup the stack to start executing the method. Expects: 292// - rdi to contain the ArtMethod 293// - rbx, r10, r11 to be available. 294// 295// Outputs 296// - rbx contains the dex registers size 297// - r11 contains the old stack pointer. 298// - \code_item is replace with a pointer to the instructions 299// - if load_ins is 1, r14 contains the ins 300.macro SETUP_STACK_FRAME code_item, refs, refs32, fp, cfi_refs, load_ins 301 FETCH_CODE_ITEM_INFO \code_item, %ebx, \refs32, %r14d, \load_ins 302 303 // Compute required frame size for dex registers: ((2 * ebx) + refs) 304 leaq (\refs, %rbx, 2), %r11 305 salq $$2, %r11 306 307 // Compute new stack pointer in r10: add 24 for saving the previous frame, 308 // pc, and method being executed. 309 leaq -24(%rsp), %r10 310 subq %r11, %r10 311 // Alignment 312 // Note: There may be two pieces of alignment but there is no need to align 313 // out args to `kPointerSize` separately before aligning to kStackAlignment. 314 andq $$-16, %r10 315 316 // Set reference and dex registers, align to pointer size for previous frame and dex pc. 317 leaq 24 + 4(%r10, \refs, 4), \refs 318 andq LITERAL(-__SIZEOF_POINTER__), \refs 319 leaq (\refs, %rbx, 4), \fp 320 321 // Now setup the stack pointer. 322 movq %rsp, %r11 323 CFI_DEF_CFA_REGISTER(r11) 324 movq %r10, %rsp 325 movq %r11, -8(\refs) 326 CFI_DEF_CFA_BREG_PLUS_UCONST \cfi_refs, -8, ((6 + 4 + 1) * 8) 327 328 // Put nulls in reference frame. 329 testl %ebx, %ebx 330 je 2f 331 movq \refs, %r10 3321: 333 movl $$0, (%r10) 334 addq $$4, %r10 335 cmpq %r10, \fp 336 jne 1b 3372: 338 // Save the ArtMethod. 339 movq %rdi, (%rsp) 340.endm 341 342// Puts the next floating point argument into the expected register, 343// fetching values based on a non-range invoke. 344// Uses rax as temporary. 345// 346// TODO: We could simplify a lot of code by loading the G argument into 347// the "inst" register. Given that we enter the handler with "1(rPC)" in 348// the rINST, we can just add rINST<<16 to the args and we don't even 349// need to pass "arg_index" around. 350.macro LOOP_OVER_SHORTY_LOADING_XMMS xmm_reg, inst, shorty, arg_index, finished 3511: // LOOP 352 movb (REG_VAR(shorty)), %al // bl := *shorty 353 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 354 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 355 je VAR(finished) 356 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 357 je 2f 358 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 359 je 3f 360 shrq MACRO_LITERAL(4), REG_VAR(inst) 361 addq MACRO_LITERAL(1), REG_VAR(arg_index) 362 // Handle extra argument in arg array taken by a long. 363 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 364 jne 1b 365 shrq MACRO_LITERAL(4), REG_VAR(inst) 366 addq MACRO_LITERAL(1), REG_VAR(arg_index) 367 jmp 1b // goto LOOP 3682: // FOUND_DOUBLE 369 subq MACRO_LITERAL(8), %rsp 370 movq REG_VAR(inst), %rax 371 andq MACRO_LITERAL(0xf), %rax 372 GET_VREG %eax, %rax 373 movl %eax, (%rsp) 374 shrq MACRO_LITERAL(4), REG_VAR(inst) 375 addq MACRO_LITERAL(1), REG_VAR(arg_index) 376 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 377 je 5f 378 movq REG_VAR(inst), %rax 379 andq MACRO_LITERAL(0xf), %rax 380 shrq MACRO_LITERAL(4), REG_VAR(inst) 381 addq MACRO_LITERAL(1), REG_VAR(arg_index) 382 jmp 6f 3835: 384 movzbl 1(rPC), %eax 385 andq MACRO_LITERAL(0xf), %rax 3866: 387 GET_VREG %eax, %rax 388 movl %eax, 4(%rsp) 389 movsd (%rsp), REG_VAR(xmm_reg) 390 addq MACRO_LITERAL(8), %rsp 391 jmp 4f 3923: // FOUND_FLOAT 393 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 394 je 7f 395 movq REG_VAR(inst), %rax 396 andq MACRO_LITERAL(0xf), %rax 397 shrq MACRO_LITERAL(4), REG_VAR(inst) 398 addq MACRO_LITERAL(1), REG_VAR(arg_index) 399 jmp 8f 4007: 401 movzbl 1(rPC), %eax 402 andq MACRO_LITERAL(0xf), %rax 4038: 404 GET_VREG_XMMs REG_VAR(xmm_reg), %rax 4054: 406.endm 407 408// Puts the next int/long/object argument in the expected register, 409// fetching values based on a non-range invoke. 410// Uses rax as temporary. 411.macro LOOP_OVER_SHORTY_LOADING_GPRS gpr_reg64, gpr_reg32, inst, shorty, arg_index, finished 4121: // LOOP 413 movb (REG_VAR(shorty)), %al // al := *shorty 414 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 415 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 416 je VAR(finished) 417 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 418 je 2f 419 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 420 je 3f 421 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 422 je 4f 423 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 424 je 7f 425 movq REG_VAR(inst), %rax 426 andq MACRO_LITERAL(0xf), %rax 427 shrq MACRO_LITERAL(4), REG_VAR(inst) 428 addq MACRO_LITERAL(1), REG_VAR(arg_index) 429 jmp 8f 4307: 431 movzbl 1(rPC), %eax 432 andq MACRO_LITERAL(0xf), %rax 4338: 434 GET_VREG REG_VAR(gpr_reg32), %rax 435 jmp 5f 4362: // FOUND_LONG 437 subq MACRO_LITERAL(8), %rsp 438 movq REG_VAR(inst), %rax 439 andq MACRO_LITERAL(0xf), %rax 440 GET_VREG %eax, %rax 441 movl %eax, (%rsp) 442 shrq MACRO_LITERAL(4), REG_VAR(inst) 443 addq MACRO_LITERAL(1), REG_VAR(arg_index) 444 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 445 je 9f 446 movq REG_VAR(inst), %rax 447 andq MACRO_LITERAL(0xf), %rax 448 shrq MACRO_LITERAL(4), REG_VAR(inst) 449 addq MACRO_LITERAL(1), REG_VAR(arg_index) 450 jmp 10f 4519: 452 movzbl 1(rPC), %eax 453 andq MACRO_LITERAL(0xf), %rax 45410: 455 GET_VREG %eax, %rax 456 movl %eax, 4(%rsp) 457 movq (%rsp), REG_VAR(gpr_reg64) 458 addq MACRO_LITERAL(8), %rsp 459 jmp 5f 4603: // SKIP_FLOAT 461 shrq MACRO_LITERAL(4), REG_VAR(inst) 462 addq MACRO_LITERAL(1), REG_VAR(arg_index) 463 jmp 1b 4644: // SKIP_DOUBLE 465 shrq MACRO_LITERAL(4), REG_VAR(inst) 466 addq MACRO_LITERAL(1), REG_VAR(arg_index) 467 cmpq MACRO_LITERAL(4), REG_VAR(arg_index) 468 je 1b 469 shrq MACRO_LITERAL(4), REG_VAR(inst) 470 addq MACRO_LITERAL(1), REG_VAR(arg_index) 471 jmp 1b 4725: 473.endm 474 475// Puts the next floating point argument into the expected register, 476// fetching values based on a range invoke. 477// Uses rax as temporary. 478.macro LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm_reg, shorty, arg_index, stack_index, finished 4791: // LOOP 480 movb (REG_VAR(shorty)), %al // al := *shorty 481 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 482 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 483 je VAR(finished) 484 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 485 je 2f 486 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 487 je 3f 488 addq MACRO_LITERAL(1), REG_VAR(arg_index) 489 addq MACRO_LITERAL(1), REG_VAR(stack_index) 490 // Handle extra argument in arg array taken by a long. 491 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 492 jne 1b 493 addq MACRO_LITERAL(1), REG_VAR(arg_index) 494 addq MACRO_LITERAL(1), REG_VAR(stack_index) 495 jmp 1b // goto LOOP 4962: // FOUND_DOUBLE 497 GET_VREG_XMMd REG_VAR(xmm_reg), REG_VAR(arg_index) 498 addq MACRO_LITERAL(2), REG_VAR(arg_index) 499 addq MACRO_LITERAL(2), REG_VAR(stack_index) 500 jmp 4f 5013: // FOUND_FLOAT 502 GET_VREG_XMMs REG_VAR(xmm_reg), REG_VAR(arg_index) 503 addq MACRO_LITERAL(1), REG_VAR(arg_index) 504 addq MACRO_LITERAL(1), REG_VAR(stack_index) 5054: 506.endm 507 508// Puts the next floating point argument into the expected stack slot, 509// fetching values based on a range invoke. 510// Uses rax as temporary. 511// 512// TODO: We could just copy all the vregs to the stack slots in a simple loop 513// (or REP MOVSD) without looking at the shorty at all. (We could also drop 514// the "stack_index" from the macros for loading registers.) We could also do 515// that conditionally if argument word count > 6; otherwise we know that all 516// args fit into registers. 517.macro LOOP_RANGE_OVER_FPs shorty, arg_index, stack_index, finished 5181: // LOOP 519 movb (REG_VAR(shorty)), %al // bl := *shorty 520 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 521 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 522 je VAR(finished) 523 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 524 je 2f 525 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 526 je 3f 527 addq MACRO_LITERAL(1), REG_VAR(arg_index) 528 addq MACRO_LITERAL(1), REG_VAR(stack_index) 529 // Handle extra argument in arg array taken by a long. 530 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 531 jne 1b 532 addq MACRO_LITERAL(1), REG_VAR(arg_index) 533 addq MACRO_LITERAL(1), REG_VAR(stack_index) 534 jmp 1b // goto LOOP 5352: // FOUND_DOUBLE 536 movq (rFP, REG_VAR(arg_index), 4), %rax 537 movq %rax, 8(%rsp, REG_VAR(stack_index), 4) 538 addq MACRO_LITERAL(2), REG_VAR(arg_index) 539 addq MACRO_LITERAL(2), REG_VAR(stack_index) 540 jmp 1b 5413: // FOUND_FLOAT 542 movl (rFP, REG_VAR(arg_index), 4), %eax 543 movl %eax, 8(%rsp, REG_VAR(stack_index), 4) 544 addq MACRO_LITERAL(1), REG_VAR(arg_index) 545 addq MACRO_LITERAL(1), REG_VAR(stack_index) 546 jmp 1b 547.endm 548 549// Puts the next int/long/object argument in the expected register, 550// fetching values based on a range invoke. 551// Uses rax as temporary. 552.macro LOOP_RANGE_OVER_SHORTY_LOADING_GPRS gpr_reg64, gpr_reg32, shorty, arg_index, stack_index, finished 5531: // LOOP 554 movb (REG_VAR(shorty)), %al // al := *shorty 555 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 556 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 557 je VAR(finished) 558 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 559 je 2f 560 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 561 je 3f 562 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 563 je 4f 564 movl (rFP, REG_VAR(arg_index), 4), REG_VAR(gpr_reg32) 565 addq MACRO_LITERAL(1), REG_VAR(arg_index) 566 addq MACRO_LITERAL(1), REG_VAR(stack_index) 567 jmp 5f 5682: // FOUND_LONG 569 movq (rFP, REG_VAR(arg_index), 4), REG_VAR(gpr_reg64) 570 addq MACRO_LITERAL(2), REG_VAR(arg_index) 571 addq MACRO_LITERAL(2), REG_VAR(stack_index) 572 jmp 5f 5733: // SKIP_FLOAT 574 addq MACRO_LITERAL(1), REG_VAR(arg_index) 575 addq MACRO_LITERAL(1), REG_VAR(stack_index) 576 jmp 1b 5774: // SKIP_DOUBLE 578 addq MACRO_LITERAL(2), REG_VAR(arg_index) 579 addq MACRO_LITERAL(2), REG_VAR(stack_index) 580 jmp 1b 5815: 582.endm 583 584// Puts the next int/long/object argument in the expected stack slot, 585// fetching values based on a range invoke. 586// Uses rax as temporary. 587.macro LOOP_RANGE_OVER_INTs shorty, arg_index, stack_index, finished 5881: // LOOP 589 movb (REG_VAR(shorty)), %al // al := *shorty 590 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 591 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 592 je VAR(finished) 593 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 594 je 2f 595 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 596 je 3f 597 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 598 je 4f 599 movl (rFP, REG_VAR(arg_index), 4), %eax 600 movl %eax, 8(%rsp, REG_VAR(stack_index), 4) 601 addq MACRO_LITERAL(1), REG_VAR(arg_index) 602 addq MACRO_LITERAL(1), REG_VAR(stack_index) 603 jmp 1b 6042: // FOUND_LONG 605 movq (rFP, REG_VAR(arg_index), 4), %rax 606 movq %rax, 8(%rsp, REG_VAR(stack_index), 4) 607 addq MACRO_LITERAL(2), REG_VAR(arg_index) 608 addq MACRO_LITERAL(2), REG_VAR(stack_index) 609 jmp 1b 6103: // SKIP_FLOAT 611 addq MACRO_LITERAL(1), REG_VAR(arg_index) 612 addq MACRO_LITERAL(1), REG_VAR(stack_index) 613 jmp 1b 6144: // SKIP_DOUBLE 615 addq MACRO_LITERAL(2), REG_VAR(arg_index) 616 addq MACRO_LITERAL(2), REG_VAR(stack_index) 617 jmp 1b 618.endm 619 620// Puts the next floating point parameter passed in physical register 621// in the expected dex register array entry. 622// Uses rax as temporary. 623.macro LOOP_OVER_SHORTY_STORING_XMMS xmm_reg, shorty, arg_index, fp, finished 6241: // LOOP 625 movb (REG_VAR(shorty)), %al // al := *shorty 626 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 627 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 628 je VAR(finished) 629 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 630 je 2f 631 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 632 je 3f 633 addq MACRO_LITERAL(1), REG_VAR(arg_index) 634 // Handle extra argument in arg array taken by a long. 635 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 636 jne 1b 637 addq MACRO_LITERAL(1), REG_VAR(arg_index) 638 jmp 1b // goto LOOP 6392: // FOUND_DOUBLE 640 movsd REG_VAR(xmm_reg),(REG_VAR(fp), REG_VAR(arg_index), 4) 641 addq MACRO_LITERAL(2), REG_VAR(arg_index) 642 jmp 4f 6433: // FOUND_FLOAT 644 movss REG_VAR(xmm_reg), (REG_VAR(fp), REG_VAR(arg_index), 4) 645 addq MACRO_LITERAL(1), REG_VAR(arg_index) 6464: 647.endm 648 649// Puts the next int/long/object parameter passed in physical register 650// in the expected dex register array entry, and in case of object in the 651// expected reference array entry. 652// Uses rax as temporary. 653.macro LOOP_OVER_SHORTY_STORING_GPRS gpr_reg64, gpr_reg32, shorty, arg_index, regs, refs, finished 6541: // LOOP 655 movb (REG_VAR(shorty)), %al // al := *shorty 656 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 657 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 658 je VAR(finished) 659 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 660 je 2f 661 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 662 je 3f 663 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 664 je 4f 665 movl REG_VAR(gpr_reg32), (REG_VAR(regs), REG_VAR(arg_index), 4) 666 cmpb MACRO_LITERAL(76), %al // if (al != 'L') goto NOT_REFERENCE 667 jne 6f 668 movl REG_VAR(gpr_reg32), (REG_VAR(refs), REG_VAR(arg_index), 4) 6696: // NOT_REFERENCE 670 addq MACRO_LITERAL(1), REG_VAR(arg_index) 671 jmp 5f 6722: // FOUND_LONG 673 movq REG_VAR(gpr_reg64), (REG_VAR(regs), REG_VAR(arg_index), 4) 674 addq MACRO_LITERAL(2), REG_VAR(arg_index) 675 jmp 5f 6763: // SKIP_FLOAT 677 addq MACRO_LITERAL(1), REG_VAR(arg_index) 678 jmp 1b 6794: // SKIP_DOUBLE 680 addq MACRO_LITERAL(2), REG_VAR(arg_index) 681 jmp 1b 6825: 683.endm 684 685// Puts the next floating point parameter passed in stack 686// in the expected dex register array entry. 687// Uses rax as temporary. 688// 689// TODO: Or we could just spill regs to the reserved slots in the caller's 690// frame and copy all regs in a simple loop. This time, however, we would 691// need to look at the shorty anyway to look for the references. 692// (The trade-off is different for passing arguments and receiving them.) 693.macro LOOP_OVER_FPs shorty, arg_index, regs, stack_ptr, finished 6941: // LOOP 695 movb (REG_VAR(shorty)), %al // al := *shorty 696 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 697 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 698 je VAR(finished) 699 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 700 je 2f 701 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 702 je 3f 703 addq MACRO_LITERAL(1), REG_VAR(arg_index) 704 // Handle extra argument in arg array taken by a long. 705 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 706 jne 1b 707 addq MACRO_LITERAL(1), REG_VAR(arg_index) 708 jmp 1b // goto LOOP 7092: // FOUND_DOUBLE 710 movq OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %rax 711 movq %rax, (REG_VAR(regs), REG_VAR(arg_index), 4) 712 addq MACRO_LITERAL(2), REG_VAR(arg_index) 713 jmp 1b 7143: // FOUND_FLOAT 715 movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %eax 716 movl %eax, (REG_VAR(regs), REG_VAR(arg_index), 4) 717 addq MACRO_LITERAL(1), REG_VAR(arg_index) 718 jmp 1b 719.endm 720 721// Puts the next int/long/object parameter passed in stack 722// in the expected dex register array entry, and in case of object in the 723// expected reference array entry. 724// Uses rax as temporary. 725.macro LOOP_OVER_INTs shorty, arg_index, regs, refs, stack_ptr, finished 7261: // LOOP 727 movb (REG_VAR(shorty)), %al // al := *shorty 728 addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ 729 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished 730 je VAR(finished) 731 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 732 je 2f 733 cmpb MACRO_LITERAL(76), %al // if (al == 'L') goto FOUND_REFERENCE 734 je 6f 735 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 736 je 3f 737 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 738 je 4f 739 movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %eax 740 movl %eax, (REG_VAR(regs), REG_VAR(arg_index), 4) 741 addq MACRO_LITERAL(1), REG_VAR(arg_index) 742 jmp 1b 7436: // FOUND_REFERENCE 744 movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %eax 745 movl %eax, (REG_VAR(regs), REG_VAR(arg_index), 4) 746 movl %eax, (REG_VAR(refs), REG_VAR(arg_index), 4) 747 addq MACRO_LITERAL(1), REG_VAR(arg_index) 748 jmp 1b 7492: // FOUND_LONG 750 movq OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %rax 751 movq %rax, (REG_VAR(regs), REG_VAR(arg_index), 4) 752 addq MACRO_LITERAL(2), REG_VAR(arg_index) 753 jmp 1b 7543: // SKIP_FLOAT 755 addq MACRO_LITERAL(1), REG_VAR(arg_index) 756 jmp 1b 7574: // SKIP_DOUBLE 758 addq MACRO_LITERAL(2), REG_VAR(arg_index) 759 jmp 1b 760.endm 761 762// Increase method hotness and do suspend check before starting executing the method. 763.macro START_EXECUTING_INSTRUCTIONS 764 movq (%rsp), %rdi 765 movzwl ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi), %esi 766#if (NTERP_HOTNESS_VALUE != 0) 767#error Expected 0 for hotness value 768#endif 769 // If the counter is at zero, handle this in the runtime. 770 testl %esi, %esi 771 je 3f 772 // Update counter. 773 addl $$-1, %esi 774 movw %si, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) 7751: 776 DO_SUSPEND_CHECK continue_label=2f 7772: 778 FETCH_INST 779 GOTO_NEXT 7803: 781 CHECK_AND_UPDATE_SHARED_MEMORY_METHOD if_hot=4f, if_not_hot=1b 7824: 783 movq $$0, %rsi 784 movq rFP, %rdx 785 call nterp_hot_method 786 jmp 2b 787.endm 788 789.macro SPILL_ALL_CALLEE_SAVES 790 PUSH r15 791 PUSH r14 792 PUSH r13 793 PUSH r12 794 PUSH rbp 795 PUSH rbx 796 SETUP_FP_CALLEE_SAVE_FRAME 797.endm 798 799.macro RESTORE_ALL_CALLEE_SAVES 800 RESTORE_FP_CALLEE_SAVE_FRAME 801 POP rbx 802 POP rbp 803 POP r12 804 POP r13 805 POP r14 806 POP r15 807.endm 808 809// Helper to setup the stack after doing a nterp to nterp call. This will setup: 810// - rNEW_FP: the new pointer to dex registers 811// - rNEW_REFS: the new pointer to references 812// - rPC: the new PC pointer to execute 813// - edi: number of arguments 814// - ecx: first dex register 815// 816// This helper expects: 817// - rax to contain the code item 818.macro SETUP_STACK_FOR_INVOKE 819 // We do the same stack overflow check as the compiler. See CanMethodUseNterp 820 // in how we limit the maximum nterp frame size. 821 testq %rax, -STACK_OVERFLOW_RESERVED_BYTES(%rsp) 822 823 // Spill all callee saves to have a consistent stack frame whether we 824 // are called by compiled code or nterp. 825 SPILL_ALL_CALLEE_SAVES 826 827 // Setup the frame. 828 SETUP_STACK_FRAME %rax, rNEW_REFS, rNEW_REFS32, rNEW_FP, CFI_NEW_REFS, load_ins=0 829 // Make r11 point to the top of the dex register array. 830 leaq (rNEW_FP, %rbx, 4), %r11 831 832 // Fetch instruction information before replacing rPC. 833 movzbl 1(rPC), %edi 834 movzwl 4(rPC), %ecx 835 836 // Set the dex pc pointer. 837 movq %rax, rPC 838 CFI_DEFINE_DEX_PC_WITH_OFFSET(CFI_TMP, CFI_DEX, 0) 839.endm 840 841// Setup arguments based on a non-range nterp to nterp call, and start executing 842// the method. We expect: 843// - rNEW_FP: the new pointer to dex registers 844// - rNEW_REFS: the new pointer to references 845// - rPC: the new PC pointer to execute 846// - edi: number of arguments 847// - ecx: first dex register 848// - r11: top of dex register array 849// - esi: receiver if non-static. 850.macro SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 851 // Now all temporary registers (except r11 containing top of registers array) 852 // are available, copy the parameters. 853 // /* op vA, vB, {vC...vG} */ 854 movl %edi, %eax 855 shrl $$4, %eax # Number of arguments 856 jz 6f # shl sets the Z flag 857 movq MACRO_LITERAL(-1), %r10 858 cmpl MACRO_LITERAL(2), %eax 859 jl 1f 860 je 2f 861 cmpl MACRO_LITERAL(4), %eax 862 jl 3f 863 je 4f 864 865 // We use a decrementing r10 to store references relative 866 // to rNEW_FP and dex registers relative to r11. 867 // 868 // TODO: We could set up r10 as the number of registers (this can be an additional output from 869 // SETUP_STACK_FOR_INVOKE) and then just decrement it by one before copying each arg to 870 // (rNEW_FP, r10, 4) and (rNEW_REFS, r10, 4). 871 // Maybe even introduce macros NEW_VREG_ADDRESS/NEW_VREG_REF_ADDRESS. 8725: 873 andq MACRO_LITERAL(15), %rdi 874 GET_VREG_OBJECT %edx, %rdi 875 movl %edx, (rNEW_FP, %r10, 4) 876 GET_VREG %edx, %rdi 877 movl %edx, (%r11, %r10, 4) 878 subq MACRO_LITERAL(1), %r10 8794: 880 movl %ecx, %eax 881 shrl MACRO_LITERAL(12), %eax 882 GET_VREG_OBJECT %edx, %rax 883 movl %edx, (rNEW_FP, %r10, 4) 884 GET_VREG %edx, %rax 885 movl %edx, (%r11, %r10, 4) 886 subq MACRO_LITERAL(1), %r10 8873: 888 movl %ecx, %eax 889 shrl MACRO_LITERAL(8), %eax 890 andl MACRO_LITERAL(0xf), %eax 891 GET_VREG_OBJECT %edx, %rax 892 movl %edx, (rNEW_FP, %r10, 4) 893 GET_VREG %edx, %rax 894 movl %edx, (%r11, %r10, 4) 895 subq MACRO_LITERAL(1), %r10 8962: 897 movl %ecx, %eax 898 shrl MACRO_LITERAL(4), %eax 899 andl MACRO_LITERAL(0xf), %eax 900 GET_VREG_OBJECT %edx, %rax 901 movl %edx, (rNEW_FP, %r10, 4) 902 GET_VREG %edx, %rax 903 movl %edx, (%r11, %r10, 4) 904 subq MACRO_LITERAL(1), %r10 9051: 906 .if \is_string_init 907 // Ignore the first argument 908 .elseif \is_static 909 movl %ecx, %eax 910 andq MACRO_LITERAL(0x000f), %rax 911 GET_VREG_OBJECT %edx, %rax 912 movl %edx, (rNEW_FP, %r10, 4) 913 GET_VREG %edx, %rax 914 movl %edx, (%r11, %r10, 4) 915 .else 916 movl %esi, (rNEW_FP, %r10, 4) 917 movl %esi, (%r11, %r10, 4) 918 .endif 919 9206: 921 // Start executing the method. 922 movq rNEW_FP, rFP 923 movq rNEW_REFS, rREFS 924 CFI_DEF_CFA_BREG_PLUS_UCONST CFI_REFS, -8, ((6 + 4 + 1) * 8) 925 START_EXECUTING_INSTRUCTIONS 926.endm 927 928// Setup arguments based on a range nterp to nterp call, and start executing 929// the method. 930.macro SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 931 // edi is number of arguments 932 // ecx is first register 933 movq MACRO_LITERAL(-4), %r10 934 .if \is_string_init 935 // Ignore the first argument 936 subl $$1, %edi 937 addl $$1, %ecx 938 .elseif !\is_static 939 subl $$1, %edi 940 addl $$1, %ecx 941 .endif 942 943 testl %edi, %edi 944 je 2f 945 leaq (rREFS, %rcx, 4), %rax # pointer to first argument in reference array 946 leaq (%rax, %rdi, 4), %rax # pointer to last argument in reference array 947 leaq (rFP, %rcx, 4), %rcx # pointer to first argument in register array 948 leaq (%rcx, %rdi, 4), %rdi # pointer to last argument in register array 949 // TODO: Same comment for copying arguments as in SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE. 9501: 951 movl -4(%rax), %edx 952 movl %edx, (rNEW_FP, %r10, 1) 953 movl -4(%rdi), %edx 954 movl %edx, (%r11, %r10, 1) 955 subq MACRO_LITERAL(4), %r10 956 subq MACRO_LITERAL(4), %rax 957 subq MACRO_LITERAL(4), %rdi 958 cmpq %rcx, %rdi 959 jne 1b 960 9612: 962 .if \is_string_init 963 // Ignore first argument 964 .elseif !\is_static 965 movl %esi, (rNEW_FP, %r10, 1) 966 movl %esi, (%r11, %r10, 1) 967 .endif 968 movq rNEW_FP, rFP 969 movq rNEW_REFS, rREFS 970 CFI_DEF_CFA_BREG_PLUS_UCONST CFI_REFS, -8, ((6 + 4 + 1) * 8) 971 START_EXECUTING_INSTRUCTIONS 972.endm 973 974.macro GET_SHORTY dest, is_interface, is_polymorphic, is_custom 975 push %rdi 976 push %rsi 977 .if \is_polymorphic 978 movq 16(%rsp), %rdi 979 movq rPC, %rsi 980 call SYMBOL(NterpGetShortyFromInvokePolymorphic) 981 .elseif \is_custom 982 movq 16(%rsp), %rdi 983 movq rPC, %rsi 984 call SYMBOL(NterpGetShortyFromInvokeCustom) 985 .elseif \is_interface 986 movq 16(%rsp), %rdi 987 movzwl 2(rPC), %esi 988 call SYMBOL(NterpGetShortyFromMethodId) 989 .else 990 call SYMBOL(NterpGetShorty) 991 .endif 992 pop %rsi 993 pop %rdi 994 movq %rax, \dest 995.endm 996 997.macro GET_SHORTY_SLOW_PATH dest, is_interface 998 // Save all registers that can hold arguments in the fast path. 999 push %rdi 1000 push %rsi 1001 push %rdx 1002 subq MACRO_LITERAL(8), %rsp 1003 mov %xmm0, (%rsp) 1004 .if \is_interface 1005 movq 32(%rsp), %rdi 1006 movzwl 2(rPC), %esi 1007 call SYMBOL(NterpGetShortyFromMethodId) 1008 .else 1009 call SYMBOL(NterpGetShorty) 1010 .endif 1011 mov (%rsp), %xmm0 1012 addq MACRO_LITERAL(8), %rsp 1013 pop %rdx 1014 pop %rsi 1015 pop %rdi 1016 movq %rax, \dest 1017.endm 1018 1019// Uses r9 as temporary. 1020.macro DO_ENTRY_POINT_CHECK call_compiled_code 1021 // On entry, the method is %rdi, the instance is %rsi 1022 leaq ExecuteNterpImpl(%rip), %r9 1023 cmpq %r9, ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) 1024 jne VAR(call_compiled_code) 1025 1026 movq ART_METHOD_DATA_OFFSET_64(%rdi), %rax 1027.endm 1028 1029// Uses r9 and r10 as temporary 1030.macro UPDATE_REGISTERS_FOR_STRING_INIT old_value, new_value 1031 movq rREFS, %r9 1032 movq rFP, %r10 10331: 1034 cmpl (%r9), \old_value 1035 jne 2f 1036 movl \new_value, (%r9) 1037 movl \new_value, (%r10) 10382: 1039 addq $$4, %r9 1040 addq $$4, %r10 1041 cmpq %r9, rFP 1042 jne 1b 1043.endm 1044 1045.macro COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, suffix="", is_string_init=0, is_polymorphic=0, is_custom=0 1046 .if \is_polymorphic 1047 // We always go to compiled code for polymorphic calls. 1048 .elseif \is_custom 1049 // We always go to compiled code for custom calls. 1050 .else 1051 DO_ENTRY_POINT_CHECK .Lcall_compiled_code_\suffix 1052 .if \is_string_init 1053 call nterp_to_nterp_string_init_non_range 1054 .elseif \is_static 1055 call nterp_to_nterp_static_non_range 1056 .else 1057 call nterp_to_nterp_instance_non_range 1058 .endif 1059 jmp .Ldone_return_\suffix 1060 .endif 1061 1062.Lcall_compiled_code_\suffix: 1063 .if \is_polymorphic 1064 // No fast path for polymorphic calls. 1065 .elseif \is_custom 1066 // No fast path for custom calls. 1067 .elseif \is_string_init 1068 // No fast path for string.init. 1069 .else 1070 testl $$ART_METHOD_NTERP_INVOKE_FAST_PATH_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi) 1071 je .Lfast_path_with_few_args_\suffix 1072 movzbl 1(rPC), %r9d 1073 movl %r9d, %ebp 1074 shrl MACRO_LITERAL(4), %ebp # Number of arguments 1075 .if \is_static 1076 jz .Linvoke_fast_path_\suffix # shl sets the Z flag 1077 .else 1078 cmpl MACRO_LITERAL(1), %ebp 1079 je .Linvoke_fast_path_\suffix 1080 .endif 1081 movzwl 4(rPC), %r11d 1082 cmpl MACRO_LITERAL(2), %ebp 1083 .if \is_static 1084 jl .Lone_arg_fast_path_\suffix 1085 .endif 1086 je .Ltwo_args_fast_path_\suffix 1087 cmpl MACRO_LITERAL(4), %ebp 1088 jl .Lthree_args_fast_path_\suffix 1089 je .Lfour_args_fast_path_\suffix 1090 1091 andl MACRO_LITERAL(0xf), %r9d 1092 GET_VREG %r9d, %r9 1093.Lfour_args_fast_path_\suffix: 1094 movl %r11d, %r8d 1095 shrl MACRO_LITERAL(12), %r8d 1096 GET_VREG %r8d, %r8 1097.Lthree_args_fast_path_\suffix: 1098 movl %r11d, %ecx 1099 shrl MACRO_LITERAL(8), %ecx 1100 andl MACRO_LITERAL(0xf), %ecx 1101 GET_VREG %ecx, %rcx 1102.Ltwo_args_fast_path_\suffix: 1103 movl %r11d, %edx 1104 shrl MACRO_LITERAL(4), %edx 1105 andl MACRO_LITERAL(0xf), %edx 1106 GET_VREG %edx, %rdx 1107.Lone_arg_fast_path_\suffix: 1108 .if \is_static 1109 andl MACRO_LITERAL(0xf), %r11d 1110 GET_VREG %esi, %r11 1111 .else 1112 // First argument already in %esi. 1113 .endif 1114.Linvoke_fast_path_\suffix: 1115 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 1116 ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 1117 1118.Lfast_path_with_few_args_\suffix: 1119 // Fast path when we have zero or one argument (modulo 'this'). If there 1120 // is one argument, we can put it in both floating point and core register. 1121 movzbl 1(rPC), %r9d 1122 shrl MACRO_LITERAL(4), %r9d # Number of arguments 1123 .if \is_static 1124 cmpl MACRO_LITERAL(1), %r9d 1125 jl .Linvoke_with_few_args_\suffix 1126 jne .Lget_shorty_\suffix 1127 movzwl 4(rPC), %r9d 1128 andl MACRO_LITERAL(0xf), %r9d // dex register of first argument 1129 GET_VREG %esi, %r9 1130 movd %esi, %xmm0 1131 .else 1132 cmpl MACRO_LITERAL(2), %r9d 1133 jl .Linvoke_with_few_args_\suffix 1134 jne .Lget_shorty_\suffix 1135 movzwl 4(rPC), %r9d 1136 shrl MACRO_LITERAL(4), %r9d 1137 andl MACRO_LITERAL(0xf), %r9d // dex register of second argument 1138 GET_VREG %edx, %r9 1139 movd %edx, %xmm0 1140 .endif 1141.Linvoke_with_few_args_\suffix: 1142 // Check if the next instruction is move-result or move-result-wide. 1143 // If it is, we fetch the shorty and jump to the regular invocation. 1144 movzwq 6(rPC), %r9 1145 andl MACRO_LITERAL(0xfe), %r9d 1146 cmpl MACRO_LITERAL(0x0a), %r9d 1147 je .Lget_shorty_and_invoke_\suffix 1148 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 1149 ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 1150.Lget_shorty_and_invoke_\suffix: 1151 .if \is_interface 1152 // Save interface method, used for conflict resolution, in a callee-save register. 1153 movq %rax, %xmm12 1154 .endif 1155 GET_SHORTY_SLOW_PATH rINSTq, \is_interface 1156 jmp .Lgpr_setup_finished_\suffix 1157 .endif 1158 1159.Lget_shorty_\suffix: 1160 .if \is_interface 1161 // Save interface method, used for conflict resolution, in a callee-save register. 1162 movq %rax, %xmm12 1163 .endif 1164 GET_SHORTY rINSTq, \is_interface, \is_polymorphic, \is_custom 1165 // From this point: 1166 // - rISNTq contains shorty (in callee-save to switch over return value after call). 1167 // - rdi contains method 1168 // - rsi contains 'this' pointer for instance method. 1169 leaq 1(rINSTq), %r9 // shorty + 1 ; ie skip return arg character 1170 movzwl 4(rPC), %r11d // arguments 1171 .if \is_string_init 1172 shrq MACRO_LITERAL(4), %r11 1173 movq $$1, %r10 // ignore first argument 1174 .elseif \is_static 1175 movq $$0, %r10 // arg_index 1176 .else 1177 shrq MACRO_LITERAL(4), %r11 1178 movq $$1, %r10 // arg_index 1179 .endif 1180 LOOP_OVER_SHORTY_LOADING_XMMS xmm0, r11, r9, r10, .Lxmm_setup_finished_\suffix 1181 LOOP_OVER_SHORTY_LOADING_XMMS xmm1, r11, r9, r10, .Lxmm_setup_finished_\suffix 1182 LOOP_OVER_SHORTY_LOADING_XMMS xmm2, r11, r9, r10, .Lxmm_setup_finished_\suffix 1183 LOOP_OVER_SHORTY_LOADING_XMMS xmm3, r11, r9, r10, .Lxmm_setup_finished_\suffix 1184 LOOP_OVER_SHORTY_LOADING_XMMS xmm4, r11, r9, r10, .Lxmm_setup_finished_\suffix 1185.Lxmm_setup_finished_\suffix: 1186 leaq 1(rINSTq), %r9 // shorty + 1 ; ie skip return arg character 1187 movzwl 4(rPC), %r11d // arguments 1188 .if \is_string_init 1189 movq $$1, %r10 // ignore first argument 1190 shrq MACRO_LITERAL(4), %r11 1191 LOOP_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r9, r10, .Lgpr_setup_finished_\suffix 1192 .elseif \is_static 1193 movq $$0, %r10 // arg_index 1194 LOOP_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r9, r10, .Lgpr_setup_finished_\suffix 1195 .else 1196 shrq MACRO_LITERAL(4), %r11 1197 movq $$1, %r10 // arg_index 1198 .endif 1199 LOOP_OVER_SHORTY_LOADING_GPRS rdx, edx, r11, r9, r10, .Lgpr_setup_finished_\suffix 1200 LOOP_OVER_SHORTY_LOADING_GPRS rcx, ecx, r11, r9, r10, .Lgpr_setup_finished_\suffix 1201 LOOP_OVER_SHORTY_LOADING_GPRS r8, r8d, r11, r9, r10, .Lgpr_setup_finished_\suffix 1202 LOOP_OVER_SHORTY_LOADING_GPRS r9, r9d, r11, r9, r10, .Lgpr_setup_finished_\suffix 1203.Lgpr_setup_finished_\suffix: 1204 .if \is_polymorphic 1205 call SYMBOL(art_quick_invoke_polymorphic) 1206 .elseif \is_custom 1207 call SYMBOL(art_quick_invoke_custom) 1208 .else 1209 .if \is_interface 1210 movq %xmm12, %rax 1211 .endif 1212 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 1213 .endif 1214 cmpb LITERAL(68), (rINSTq) // Test if result type char == 'D'. 1215 je .Lreturn_double_\suffix 1216 cmpb LITERAL(70), (rINSTq) // Test if result type char == 'F'. 1217 jne .Ldone_return_\suffix 1218.Lreturn_float_\suffix: 1219 movd %xmm0, %eax 1220 jmp .Ldone_return_\suffix 1221.Lreturn_double_\suffix: 1222 movq %xmm0, %rax 1223.Ldone_return_\suffix: 1224 /* resume execution of caller */ 1225 .if \is_string_init 1226 movzwl 4(rPC), %r11d // arguments 1227 andq $$0xf, %r11 1228 GET_VREG %esi, %r11 1229 UPDATE_REGISTERS_FOR_STRING_INIT %esi, %eax 1230 .endif 1231 1232 .if \is_polymorphic 1233 ADVANCE_PC_FETCH_AND_GOTO_NEXT 4 1234 .else 1235 ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 1236 .endif 1237.endm 1238 1239.macro COMMON_INVOKE_RANGE is_static=0, is_interface=0, suffix="", is_string_init=0, is_polymorphic=0, is_custom=0 1240 .if \is_polymorphic 1241 // We always go to compiled code for polymorphic calls. 1242 .elseif \is_custom 1243 // We always go to compiled code for custom calls. 1244 .else 1245 DO_ENTRY_POINT_CHECK .Lcall_compiled_code_range_\suffix 1246 .if \is_string_init 1247 call nterp_to_nterp_string_init_range 1248 .elseif \is_static 1249 call nterp_to_nterp_static_range 1250 .else 1251 call nterp_to_nterp_instance_range 1252 .endif 1253 jmp .Ldone_return_range_\suffix 1254 .endif 1255 1256.Lcall_compiled_code_range_\suffix: 1257 .if \is_polymorphic 1258 // No fast path for polymorphic calls. 1259 .elseif \is_custom 1260 // No fast path for custom calls. 1261 .elseif \is_string_init 1262 // No fast path for string.init. 1263 .else 1264 testl $$ART_METHOD_NTERP_INVOKE_FAST_PATH_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi) 1265 je .Lfast_path_with_few_args_range_\suffix 1266 movzbl 1(rPC), %r9d // number of arguments 1267 .if \is_static 1268 testl %r9d, %r9d 1269 je .Linvoke_fast_path_range_\suffix 1270 .else 1271 cmpl MACRO_LITERAL(1), %r9d 1272 je .Linvoke_fast_path_range_\suffix 1273 .endif 1274 movzwl 4(rPC), %r11d // dex register of first argument 1275 leaq (rFP, %r11, 4), %r11 // location of first dex register value 1276 cmpl MACRO_LITERAL(2), %r9d 1277 .if \is_static 1278 jl .Lone_arg_fast_path_range_\suffix 1279 .endif 1280 je .Ltwo_args_fast_path_range_\suffix 1281 cmp MACRO_LITERAL(4), %r9d 1282 jl .Lthree_args_fast_path_range_\suffix 1283 je .Lfour_args_fast_path_range_\suffix 1284 cmp MACRO_LITERAL(5), %r9d 1285 je .Lfive_args_fast_path_range_\suffix 1286 1287.Lloop_over_fast_path_range_\suffix: 1288 subl MACRO_LITERAL(1), %r9d 1289 movl (%r11, %r9, 4), %r8d 1290 movl %r8d, 8(%rsp, %r9, 4) // Add 8 for the ArtMethod 1291 cmpl MACRO_LITERAL(5), %r9d 1292 jne .Lloop_over_fast_path_range_\suffix 1293 1294.Lfive_args_fast_path_range_\suffix: 1295 movl 16(%r11), %r9d 1296.Lfour_args_fast_path_range_\suffix: 1297 movl 12(%r11), %r8d 1298.Lthree_args_fast_path_range_\suffix: 1299 movl 8(%r11), %ecx 1300.Ltwo_args_fast_path_range_\suffix: 1301 movl 4(%r11), %edx 1302.Lone_arg_fast_path_range_\suffix: 1303 .if \is_static 1304 movl 0(%r11), %esi 1305 .else 1306 // First argument already in %esi. 1307 .endif 1308.Linvoke_fast_path_range_\suffix: 1309 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 1310 ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 1311 1312.Lfast_path_with_few_args_range_\suffix: 1313 // Fast path when we have zero or one argument (modulo 'this'). If there 1314 // is one argument, we can put it in both floating point and core register. 1315 movzbl 1(rPC), %r9d # Number of arguments 1316 .if \is_static 1317 cmpl MACRO_LITERAL(1), %r9d 1318 jl .Linvoke_with_few_args_range_\suffix 1319 jne .Lget_shorty_range_\suffix 1320 movzwl 4(rPC), %r9d // Dex register of first argument 1321 GET_VREG %esi, %r9 1322 movd %esi, %xmm0 1323 .else 1324 cmpl MACRO_LITERAL(2), %r9d 1325 jl .Linvoke_with_few_args_range_\suffix 1326 jne .Lget_shorty_range_\suffix 1327 movzwl 4(rPC), %r9d 1328 addl MACRO_LITERAL(1), %r9d // dex register of second argument 1329 GET_VREG %edx, %r9 1330 movd %edx, %xmm0 1331 .endif 1332.Linvoke_with_few_args_range_\suffix: 1333 // Check if the next instruction is move-result or move-result-wide. 1334 // If it is, we fetch the shorty and jump to the regular invocation. 1335 movzwq 6(rPC), %r9 1336 and MACRO_LITERAL(0xfe), %r9d 1337 cmpl MACRO_LITERAL(0x0a), %r9d 1338 je .Lget_shorty_and_invoke_range_\suffix 1339 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 1340 ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 1341.Lget_shorty_and_invoke_range_\suffix: 1342 .if \is_interface 1343 // Save interface method, used for conflict resolution, in a callee-save register. 1344 movq %rax, %xmm12 1345 .endif 1346 GET_SHORTY_SLOW_PATH rINSTq, \is_interface 1347 jmp .Lgpr_setup_finished_range_\suffix 1348 .endif 1349 1350.Lget_shorty_range_\suffix: 1351 .if \is_interface 1352 // Save interface method, used for conflict resolution, in a callee-saved register. 1353 movq %rax, %xmm12 1354 .endif 1355 GET_SHORTY rINSTq, \is_interface, \is_polymorphic, \is_custom 1356 // From this point: 1357 // - rINSTq contains shorty (in callee-save to switch over return value after call). 1358 // - rdi contains method 1359 // - rsi contains 'this' pointer for instance method. 1360 leaq 1(rINSTq), %r9 // shorty + 1 ; ie skip return arg character 1361 movzwl 4(rPC), %r10d // arg start index 1362 .if \is_string_init 1363 addq $$1, %r10 // arg start index 1364 movq $$1, %rbp // index in stack 1365 .elseif \is_static 1366 movq $$0, %rbp // index in stack 1367 .else 1368 addq $$1, %r10 // arg start index 1369 movq $$1, %rbp // index in stack 1370 .endif 1371 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm0, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1372 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm1, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1373 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm2, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1374 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm3, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1375 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm4, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1376 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm5, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1377 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm6, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1378 LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm7, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1379 LOOP_RANGE_OVER_FPs r9, r10, rbp, .Lxmm_setup_finished_range_\suffix 1380.Lxmm_setup_finished_range_\suffix: 1381 leaq 1(%rbx), %r11 // shorty + 1 ; ie skip return arg character 1382 movzwl 4(rPC), %r10d // arg start index 1383 .if \is_string_init 1384 addq $$1, %r10 // arg start index 1385 movq $$1, %rbp // index in stack 1386 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1387 .elseif \is_static 1388 movq $$0, %rbp // index in stack 1389 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1390 .else 1391 addq $$1, %r10 // arg start index 1392 movq $$1, %rbp // index in stack 1393 .endif 1394 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rdx, edx, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1395 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rcx, ecx, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1396 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS r8, r8d, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1397 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS r9, r9d, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1398 LOOP_RANGE_OVER_INTs r11, r10, rbp, .Lgpr_setup_finished_range_\suffix 1399 1400.Lgpr_setup_finished_range_\suffix: 1401 .if \is_polymorphic 1402 call SYMBOL(art_quick_invoke_polymorphic) 1403 .elseif \is_custom 1404 call SYMBOL(art_quick_invoke_custom) 1405 .else 1406 .if \is_interface 1407 // Set the hidden argument for conflict resolution. 1408 movq %xmm12, %rax 1409 .endif 1410 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 1411 .endif 1412 cmpb LITERAL(68), (%rbx) // Test if result type char == 'D'. 1413 je .Lreturn_range_double_\suffix 1414 cmpb LITERAL(70), (%rbx) // Test if result type char == 'F'. 1415 je .Lreturn_range_float_\suffix 1416 /* resume execution of caller */ 1417.Ldone_return_range_\suffix: 1418 .if \is_string_init 1419 movzwl 4(rPC), %r11d // arguments 1420 GET_VREG %esi, %r11 1421 UPDATE_REGISTERS_FOR_STRING_INIT %esi, %eax 1422 .endif 1423 1424 .if \is_polymorphic 1425 ADVANCE_PC_FETCH_AND_GOTO_NEXT 4 1426 .else 1427 ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 1428 .endif 1429.Lreturn_range_double_\suffix: 1430 movq %xmm0, %rax 1431 jmp .Ldone_return_range_\suffix 1432.Lreturn_range_float_\suffix: 1433 movd %xmm0, %eax 1434 jmp .Ldone_return_range_\suffix 1435.endm 1436 1437// Helper for static field get. 1438.macro OP_SGET load="movl", wide="0" 1439 // Fast-path which gets the field from thread-local cache. 1440% fetch_from_thread_cache("%rax", miss_label="2f") 14411: 1442 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 1443 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 1444 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1445 jne 3f 14464: 1447 .if \wide 1448 movq (%eax,%edx,1), %rax 1449 SET_WIDE_VREG %rax, rINSTq # fp[A] <- value 1450 .else 1451 \load (%eax, %edx, 1), %eax 1452 SET_VREG %eax, rINSTq # fp[A] <- value 1453 .endif 1454 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 14552: 1456 EXPORT_PC 1457 movq rSELF:THREAD_SELF_OFFSET, %rdi 1458 movq 0(%rsp), %rsi 1459 movq rPC, %rdx 1460 movq $$0, %rcx 1461 call nterp_get_static_field 1462 // Clear the marker that we put for volatile fields. The x86 memory 1463 // model doesn't require a barrier. 1464 andq $$-2, %rax 1465 jmp 1b 14663: 1467 call art_quick_read_barrier_mark_reg00 1468 jmp 4b 1469.endm 1470 1471// Helper for static field put. 1472.macro OP_SPUT rINST_reg="rINST", store="movl", wide="0": 1473 // Fast-path which gets the field from thread-local cache. 1474% fetch_from_thread_cache("%rax", miss_label="2f") 14751: 1476 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 1477 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 1478 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1479 jne 3f 14804: 1481 .if \wide 1482 GET_WIDE_VREG rINSTq, rINSTq # rINST <- v[A] 1483 .else 1484 GET_VREG rINST, rINSTq # rINST <- v[A] 1485 .endif 1486 \store \rINST_reg, (%rax,%rdx,1) 1487 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 14882: 1489 EXPORT_PC 1490 movq rSELF:THREAD_SELF_OFFSET, %rdi 1491 movq 0(%rsp), %rsi 1492 movq rPC, %rdx 1493 movq $$0, %rcx 1494 call nterp_get_static_field 1495 testq MACRO_LITERAL(1), %rax 1496 je 1b 1497 // Clear the marker that we put for volatile fields. The x86 memory 1498 // model doesn't require a barrier. 1499 CLEAR_VOLATILE_MARKER %rax 1500 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 1501 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 1502 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1503 jne 6f 15045: 1505 .if \wide 1506 GET_WIDE_VREG rINSTq, rINSTq # rINST <- v[A] 1507 .else 1508 GET_VREG rINST, rINSTq # rINST <- v[A] 1509 .endif 1510 \store \rINST_reg, (%rax,%rdx,1) 1511 lock addl $$0, (%rsp) 1512 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 15133: 1514 call art_quick_read_barrier_mark_reg00 1515 jmp 4b 15166: 1517 call art_quick_read_barrier_mark_reg00 1518 jmp 5b 1519.endm 1520 1521 1522.macro OP_IPUT_INTERNAL rINST_reg="rINST", store="movl", wide="0": 1523 movzbq rINSTbl, %rcx # rcx <- BA 1524 sarl $$4, %ecx # ecx <- B 1525 GET_VREG %ecx, %rcx # vB (object we're operating on) 1526 testl %ecx, %ecx # is object null? 1527 je common_errNullObject 1528 andb $$0xf, rINSTbl # rINST <- A 1529 .if \wide 1530 GET_WIDE_VREG rINSTq, rINSTq # rax<- fp[A]/fp[A+1] 1531 .else 1532 GET_VREG rINST, rINSTq # rINST <- v[A] 1533 .endif 1534 \store \rINST_reg, (%rcx,%rax,1) 1535.endm 1536 1537// Helper for instance field put. 1538.macro OP_IPUT rINST_reg="rINST", store="movl", wide="0": 1539 // Fast-path which gets the field from thread-local cache. 1540% fetch_from_thread_cache("%rax", miss_label="2f") 15411: 1542 OP_IPUT_INTERNAL \rINST_reg, \store, \wide 1543 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 15442: 1545 EXPORT_PC 1546 movq rSELF:THREAD_SELF_OFFSET, %rdi 1547 movq 0(%rsp), %rsi 1548 movq rPC, %rdx 1549 movq $$0, %rcx 1550 call nterp_get_instance_field_offset 1551 testl %eax, %eax 1552 jns 1b 1553 negl %eax 1554 OP_IPUT_INTERNAL \rINST_reg, \store, \wide 1555 lock addl $$0, (%rsp) 1556 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 1557.endm 1558 1559// Helper for instance field get. 1560.macro OP_IGET load="movl", wide="0" 1561 // Fast-path which gets the field from thread-local cache. 1562% fetch_from_thread_cache("%rax", miss_label="2f") 15631: 1564 movl rINST, %ecx # rcx <- BA 1565 sarl $$4, %ecx # ecx <- B 1566 GET_VREG %ecx, %rcx # vB (object we're operating on) 1567 testl %ecx, %ecx # is object null? 1568 je common_errNullObject 1569 andb $$0xf,rINSTbl # rINST <- A 1570 .if \wide 1571 movq (%rcx,%rax,1), %rax 1572 SET_WIDE_VREG %rax, rINSTq # fp[A] <- value 1573 .else 1574 \load (%rcx,%rax,1), %eax 1575 SET_VREG %eax, rINSTq # fp[A] <- value 1576 .endif 1577 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 15782: 1579 EXPORT_PC 1580 movq rSELF:THREAD_SELF_OFFSET, %rdi 1581 movq 0(%rsp), %rsi 1582 movq rPC, %rdx 1583 movq $$0, %rcx 1584 call nterp_get_instance_field_offset 1585 testl %eax, %eax 1586 jns 1b 1587 negl %eax 1588 jmp 1b 1589.endm 1590 1591.macro SETUP_REFERENCE_PARAMETER_IN_GPR gpr32, regs, refs, ins, arg_offset, finished 1592 movl REG_VAR(gpr32), (REG_VAR(regs), REG_VAR(arg_offset)) 1593 movl REG_VAR(gpr32), (REG_VAR(refs), REG_VAR(arg_offset)) 1594 addq MACRO_LITERAL(4), REG_VAR(arg_offset) 1595 subl MACRO_LITERAL(1), REG_VAR(ins) 1596 je \finished 1597.endm 1598 1599// Uses eax as temporary 1600.macro SETUP_REFERENCE_PARAMETERS_IN_STACK regs, refs, ins, stack_ptr, arg_offset 16011: 1602 movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_offset)), %eax 1603 movl %eax, (REG_VAR(regs), REG_VAR(arg_offset)) 1604 movl %eax, (REG_VAR(refs), REG_VAR(arg_offset)) 1605 addq MACRO_LITERAL(4), REG_VAR(arg_offset) 1606 subl MACRO_LITERAL(1), REG_VAR(ins) 1607 jne 1b 1608.endm 1609 1610.macro CHECK_AND_UPDATE_SHARED_MEMORY_METHOD if_hot, if_not_hot 1611 testl $$ART_METHOD_IS_MEMORY_SHARED_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi) 1612 jz \if_hot 1613 // Intrinsics are always in the boot image and considered hot. 1614 testl $$ART_METHOD_IS_INTRINSIC_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi) 1615 jnz \if_hot 1616 movzwl rSELF:THREAD_SHARED_METHOD_HOTNESS_OFFSET, %esi 1617 testl %esi, %esi 1618 je \if_hot 1619 addl $$-1, %esi 1620 movw %si, rSELF:THREAD_SHARED_METHOD_HOTNESS_OFFSET 1621 jmp \if_not_hot 1622.endm 1623 1624.macro DO_SUSPEND_CHECK continue_label 1625 testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET 1626 jz \continue_label 1627 EXPORT_PC 1628 call SYMBOL(art_quick_test_suspend) 1629.endm 1630 1631%def entry(): 1632/* 1633 * ArtMethod entry point. 1634 * 1635 * On entry: 1636 * rdi ArtMethod* callee 1637 * rest method parameters 1638 */ 1639 1640OAT_ENTRY ExecuteNterpWithClinitImpl 1641 .cfi_startproc 1642 // For simplicity, we don't do a read barrier here, but instead rely 1643 // on art_quick_resolution_trampoline to always have a suspend point before 1644 // calling back here. 1645 movl ART_METHOD_DECLARING_CLASS_OFFSET(%rdi), %r10d 1646 cmpl $$(MIRROR_CLASS_STATUS_VISIBLY_INITIALIZED_SHIFTED), MIRROR_CLASS_STATUS_OFFSET(%r10d) 1647 jae ExecuteNterpImpl 1648 cmpl $$(MIRROR_CLASS_STATUS_INITIALIZING_SHIFTED), MIRROR_CLASS_STATUS_OFFSET(%r10d) 1649 jb art_quick_resolution_trampoline 1650 movl MIRROR_CLASS_CLINIT_THREAD_ID_OFFSET(%r10d), %r10d 1651 cmpl %r10d, rSELF:THREAD_TID_OFFSET 1652 je ExecuteNterpImpl 1653 jmp art_quick_resolution_trampoline 1654 .cfi_endproc 1655 .global SYMBOL(EndExecuteNterpWithClinitImpl) 1656SYMBOL(EndExecuteNterpWithClinitImpl): 1657 1658OAT_ENTRY ExecuteNterpImpl 1659 .cfi_startproc 1660 .cfi_def_cfa rsp, 8 1661 testq %rax, -STACK_OVERFLOW_RESERVED_BYTES(%rsp) 1662 /* Spill callee save regs */ 1663 SPILL_ALL_CALLEE_SAVES 1664 1665 movq ART_METHOD_DATA_OFFSET_64(%rdi), rPC 1666 1667 // Setup the stack for executing the method. 1668 SETUP_STACK_FRAME rPC, rREFS, rREFS32, rFP, CFI_REFS, load_ins=1 1669 1670 // Setup the parameters 1671 testl %r14d, %r14d 1672 je .Lxmm_setup_finished 1673 1674 subq %r14, %rbx 1675 salq $$2, %rbx // rbx is now the offset for inputs into the registers array. 1676 1677 testl $$ART_METHOD_NTERP_ENTRY_POINT_FAST_PATH_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi) 1678 je .Lsetup_slow_path 1679 leaq (rFP, %rbx, 1), %rdi 1680 leaq (rREFS, %rbx, 1), %rbx 1681 movq $$0, %r10 1682 1683 SETUP_REFERENCE_PARAMETER_IN_GPR esi, rdi, rbx, r14d, r10, .Lxmm_setup_finished 1684 SETUP_REFERENCE_PARAMETER_IN_GPR edx, rdi, rbx, r14d, r10, .Lxmm_setup_finished 1685 SETUP_REFERENCE_PARAMETER_IN_GPR ecx, rdi, rbx, r14d, r10, .Lxmm_setup_finished 1686 SETUP_REFERENCE_PARAMETER_IN_GPR r8d, rdi, rbx, r14d, r10, .Lxmm_setup_finished 1687 SETUP_REFERENCE_PARAMETER_IN_GPR r9d, rdi, rbx, r14d, r10, .Lxmm_setup_finished 1688 SETUP_REFERENCE_PARAMETERS_IN_STACK rdi, rbx, r14d, r11, r10 1689 jmp .Lxmm_setup_finished 1690 1691.Lsetup_slow_path: 1692 // If the method is not static and there is one argument ('this'), we don't need to fetch the 1693 // shorty. 1694 testl $$ART_METHOD_IS_STATIC_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi) 1695 jne .Lsetup_with_shorty 1696 1697 movl %esi, (rFP, %rbx) 1698 movl %esi, (rREFS, %rbx) 1699 1700 cmpl $$1, %r14d 1701 je .Lxmm_setup_finished 1702 1703.Lsetup_with_shorty: 1704 // TODO: Get shorty in a better way and remove below 1705 push %rdi 1706 push %rsi 1707 push %rdx 1708 push %rcx 1709 push %r8 1710 push %r9 1711 1712 // Save xmm registers + alignment. 1713 subq MACRO_LITERAL(8 * 8 + 8), %rsp 1714 movq %xmm0, 0(%rsp) 1715 movq %xmm1, 8(%rsp) 1716 movq %xmm2, 16(%rsp) 1717 movq %xmm3, 24(%rsp) 1718 movq %xmm4, 32(%rsp) 1719 movq %xmm5, 40(%rsp) 1720 movq %xmm6, 48(%rsp) 1721 movq %xmm7, 56(%rsp) 1722 1723 call SYMBOL(NterpGetShorty) 1724 // Save shorty in callee-save rbp. 1725 movq %rax, %rbp 1726 1727 // Restore xmm registers + alignment. 1728 movq 0(%rsp), %xmm0 1729 movq 8(%rsp), %xmm1 1730 movq 16(%rsp), %xmm2 1731 movq 24(%rsp), %xmm3 1732 movq 32(%rsp), %xmm4 1733 movq 40(%rsp), %xmm5 1734 movq 48(%rsp), %xmm6 1735 movq 56(%rsp), %xmm7 1736 addq MACRO_LITERAL(8 * 8 + 8), %rsp 1737 1738 pop %r9 1739 pop %r8 1740 pop %rcx 1741 pop %rdx 1742 pop %rsi 1743 pop %rdi 1744 // Reload the old stack pointer, which used to be stored in %r11, which is not callee-saved. 1745 movq -8(rREFS), %r11 1746 // TODO: Get shorty in a better way and remove above 1747 1748 movq $$0, %r14 1749 testl $$ART_METHOD_IS_STATIC_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi) 1750 1751 // Available: rdi, r10 1752 // Note the leaq below don't change the flags. 1753 leaq 1(%rbp), %r10 // shorty + 1 ; ie skip return arg character 1754 leaq (rFP, %rbx, 1), %rdi 1755 leaq (rREFS, %rbx, 1), %rbx 1756 jne .Lhandle_static_method 1757 addq $$4, %rdi 1758 addq $$4, %rbx 1759 addq $$4, %r11 1760 jmp .Lcontinue_setup_gprs 1761.Lhandle_static_method: 1762 LOOP_OVER_SHORTY_STORING_GPRS rsi, esi, r10, r14, rdi, rbx, .Lgpr_setup_finished 1763.Lcontinue_setup_gprs: 1764 LOOP_OVER_SHORTY_STORING_GPRS rdx, edx, r10, r14, rdi, rbx, .Lgpr_setup_finished 1765 LOOP_OVER_SHORTY_STORING_GPRS rcx, ecx, r10, r14, rdi, rbx, .Lgpr_setup_finished 1766 LOOP_OVER_SHORTY_STORING_GPRS r8, r8d, r10, r14, rdi, rbx, .Lgpr_setup_finished 1767 LOOP_OVER_SHORTY_STORING_GPRS r9, r9d, r10, r14, rdi, rbx, .Lgpr_setup_finished 1768 LOOP_OVER_INTs r10, r14, rdi, rbx, r11, .Lgpr_setup_finished 1769.Lgpr_setup_finished: 1770 leaq 1(%rbp), %r10 // shorty + 1 ; ie skip return arg character 1771 movq $$0, %r14 // reset counter 1772 LOOP_OVER_SHORTY_STORING_XMMS xmm0, r10, r14, rdi, .Lxmm_setup_finished 1773 LOOP_OVER_SHORTY_STORING_XMMS xmm1, r10, r14, rdi, .Lxmm_setup_finished 1774 LOOP_OVER_SHORTY_STORING_XMMS xmm2, r10, r14, rdi, .Lxmm_setup_finished 1775 LOOP_OVER_SHORTY_STORING_XMMS xmm3, r10, r14, rdi, .Lxmm_setup_finished 1776 LOOP_OVER_SHORTY_STORING_XMMS xmm4, r10, r14, rdi, .Lxmm_setup_finished 1777 LOOP_OVER_SHORTY_STORING_XMMS xmm5, r10, r14, rdi, .Lxmm_setup_finished 1778 LOOP_OVER_SHORTY_STORING_XMMS xmm6, r10, r14, rdi, .Lxmm_setup_finished 1779 LOOP_OVER_SHORTY_STORING_XMMS xmm7, r10, r14, rdi, .Lxmm_setup_finished 1780 LOOP_OVER_FPs r10, r14, rdi, r11, .Lxmm_setup_finished 1781.Lxmm_setup_finished: 1782 CFI_DEFINE_DEX_PC_WITH_OFFSET(CFI_TMP, CFI_DEX, 0) 1783 1784 // Set rIBASE 1785 leaq artNterpAsmInstructionStart(%rip), rIBASE 1786 /* start executing the instruction at rPC */ 1787 START_EXECUTING_INSTRUCTIONS 1788 /* NOTE: no fallthrough */ 1789 // cfi info continues, and covers the whole nterp implementation. 1790 END ExecuteNterpImpl 1791 1792%def opcode_pre(): 1793 1794%def fetch_from_thread_cache(dest_reg, miss_label): 1795 // Fetch some information from the thread cache. 1796 // Uses rax, rdx, rcx as temporaries. 1797 movq rSELF:THREAD_SELF_OFFSET, %rax 1798 movq rPC, %rdx 1799 salq MACRO_LITERAL(THREAD_INTERPRETER_CACHE_SIZE_SHIFT), %rdx 1800 andq MACRO_LITERAL(THREAD_INTERPRETER_CACHE_SIZE_MASK), %rdx 1801 cmpq THREAD_INTERPRETER_CACHE_OFFSET(%rax, %rdx, 1), rPC 1802 jne ${miss_label} 1803 movq __SIZEOF_POINTER__+THREAD_INTERPRETER_CACHE_OFFSET(%rax, %rdx, 1), ${dest_reg} 1804 1805%def footer(): 1806/* 1807 * =========================================================================== 1808 * Common subroutines and data 1809 * =========================================================================== 1810 */ 1811 1812 .text 1813 .align 2 1814 1815// Enclose all code below in a symbol (which gets printed in backtraces). 1816ENTRY nterp_helper 1817 1818// Note: mterp also uses the common_* names below for helpers, but that's OK 1819// as the C compiler compiled each interpreter separately. 1820common_errDivideByZero: 1821 EXPORT_PC 1822 call art_quick_throw_div_zero 1823 1824// Expect array in edi, index in esi. 1825common_errArrayIndex: 1826 EXPORT_PC 1827 movl MIRROR_ARRAY_LENGTH_OFFSET(%edi), %eax 1828 movl %esi, %edi 1829 movl %eax, %esi 1830 call art_quick_throw_array_bounds 1831 1832common_errNullObject: 1833 EXPORT_PC 1834 call art_quick_throw_null_pointer_exception 1835 1836NterpCommonInvokeStatic: 1837 COMMON_INVOKE_NON_RANGE is_static=1, is_interface=0, suffix="invokeStatic" 1838 1839NterpCommonInvokeStaticRange: 1840 COMMON_INVOKE_RANGE is_static=1, is_interface=0, suffix="invokeStatic" 1841 1842NterpCommonInvokeInstance: 1843 COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, suffix="invokeInstance" 1844 1845NterpCommonInvokeInstanceRange: 1846 COMMON_INVOKE_RANGE is_static=0, is_interface=0, suffix="invokeInstance" 1847 1848NterpCommonInvokeInterface: 1849 COMMON_INVOKE_NON_RANGE is_static=0, is_interface=1, suffix="invokeInterface" 1850 1851NterpCommonInvokeInterfaceRange: 1852 COMMON_INVOKE_RANGE is_static=0, is_interface=1, suffix="invokeInterface" 1853 1854NterpCommonInvokePolymorphic: 1855 COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, is_string_init=0, is_polymorphic=1, suffix="invokePolymorphic" 1856 1857NterpCommonInvokePolymorphicRange: 1858 COMMON_INVOKE_RANGE is_static=0, is_interface=0, is_polymorphic=1, suffix="invokePolymorphic" 1859 1860NterpCommonInvokeCustom: 1861 COMMON_INVOKE_NON_RANGE is_static=1, is_interface=0, is_string_init=0, is_polymorphic=0, is_custom=1, suffix="invokeCustom" 1862 1863NterpCommonInvokeCustomRange: 1864 COMMON_INVOKE_RANGE is_static=1, is_interface=0, is_polymorphic=0, is_custom=1, suffix="invokeCustom" 1865 1866NterpHandleStringInit: 1867 COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, is_string_init=1, suffix="stringInit" 1868 1869NterpHandleStringInitRange: 1870 COMMON_INVOKE_RANGE is_static=0, is_interface=0, is_string_init=1, suffix="stringInit" 1871 1872NterpNewInstance: 1873 EXPORT_PC 1874 // Fast-path which gets the class from thread-local cache. 1875% fetch_from_thread_cache("%rdi", miss_label="2f") 1876 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1877 jne 3f 18784: 1879 callq *rSELF:THREAD_ALLOC_OBJECT_ENTRYPOINT_OFFSET 18801: 1881 SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value 1882 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 18832: 1884 movq rSELF:THREAD_SELF_OFFSET, %rdi 1885 movq 0(%rsp), %rsi 1886 movq rPC, %rdx 1887 call nterp_allocate_object 1888 jmp 1b 18893: 1890 // 07 is %rdi 1891 call art_quick_read_barrier_mark_reg07 1892 jmp 4b 1893 1894NterpNewArray: 1895 /* new-array vA, vB, class@CCCC */ 1896 EXPORT_PC 1897 // Fast-path which gets the class from thread-local cache. 1898% fetch_from_thread_cache("%rdi", miss_label="2f") 1899 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 1900 jne 3f 19011: 1902 movzbl rINSTbl,%esi 1903 sarl $$4,%esi # esi<- B 1904 GET_VREG %esi %rsi # esi<- vB (array length) 1905 andb $$0xf,rINSTbl # rINST<- A 1906 callq *rSELF:THREAD_ALLOC_ARRAY_ENTRYPOINT_OFFSET 1907 SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value 1908 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 19092: 1910 movq rSELF:THREAD_SELF_OFFSET, %rdi 1911 movq 0(%rsp), %rsi 1912 movq rPC, %rdx 1913 call nterp_get_class 1914 movq %rax, %rdi 1915 jmp 1b 19163: 1917 // 07 is %rdi 1918 call art_quick_read_barrier_mark_reg07 1919 jmp 1b 1920 1921NterpPutObjectInstanceField: 1922 movl rINST, %ebp # rbp <- BA 1923 andl $$0xf, %ebp # rbp <- A 1924 GET_VREG %ecx, %rbp # ecx <- v[A] 1925 sarl $$4, rINST 1926 // Fast-path which gets the field from thread-local cache. 1927% fetch_from_thread_cache("%rax", miss_label="2f") 19281: 1929 GET_VREG rINST, rINSTq # vB (object we're operating on) 1930 testl rINST, rINST # is object null? 1931 je common_errNullObject 1932 POISON_HEAP_REF ecx 1933 movl %ecx, (rINSTq,%rax,1) 1934 testl %ecx, %ecx 1935 je 4f 1936 movq rSELF:THREAD_CARD_TABLE_OFFSET, %rax 1937 shrq $$CARD_TABLE_CARD_SHIFT, rINSTq 1938 movb %al, (%rax, rINSTq, 1) 19394: 1940 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 19412: 1942 EXPORT_PC 1943 movq rSELF:THREAD_SELF_OFFSET, %rdi 1944 movq 0(%rsp), %rsi 1945 movq rPC, %rdx 1946 // %rcx is already set. 1947 call nterp_get_instance_field_offset 1948 // Reload the value as it may have moved. 1949 GET_VREG %ecx, %rbp # ecx <- v[A] 1950 testl %eax, %eax 1951 jns 1b 1952 GET_VREG rINST, rINSTq # vB (object we're operating on) 1953 testl rINST, rINST # is object null? 1954 je common_errNullObject 1955 negl %eax 1956 POISON_HEAP_REF ecx 1957 movl %ecx, (rINSTq,%rax,1) 1958 testl %ecx, %ecx 1959 je 5f 1960 movq rSELF:THREAD_CARD_TABLE_OFFSET, %rax 1961 shrq $$CARD_TABLE_CARD_SHIFT, rINSTq 1962 movb %al, (%rax, rINSTq, 1) 19635: 1964 lock addl $$0, (%rsp) 1965 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 1966 1967NterpGetObjectInstanceField: 1968 // Fast-path which gets the field from thread-local cache. 1969% fetch_from_thread_cache("%rax", miss_label="2f") 19701: 1971 movl rINST, %ecx # rcx <- BA 1972 sarl $$4, %ecx # ecx <- B 1973 GET_VREG %ecx, %rcx # vB (object we're operating on) 1974 testl %ecx, %ecx # is object null? 1975 je common_errNullObject 1976 testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%ecx) 1977 movl (%rcx,%rax,1), %eax 1978 jnz 3f 1979 UNPOISON_HEAP_REF eax // Affects flags, so we cannot unpoison before the jnz. 19804: 1981 andb $$0xf,rINSTbl # rINST <- A 1982 SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value 1983 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 19842: 1985 EXPORT_PC 1986 movq rSELF:THREAD_SELF_OFFSET, %rdi 1987 movq 0(%rsp), %rsi 1988 movq rPC, %rdx 1989 movq $$0, %rcx 1990 call nterp_get_instance_field_offset 1991 testl %eax, %eax 1992 jns 1b 1993 // For volatile fields, we return a negative offset. Remove the sign 1994 // and no need for any barrier thanks to the memory model. 1995 negl %eax 1996 jmp 1b 19973: 1998 UNPOISON_HEAP_REF eax 1999 // reg00 is eax 2000 call art_quick_read_barrier_mark_reg00 2001 jmp 4b 2002 2003NterpPutObjectStaticField: 2004 GET_VREG %ebp, rINSTq 2005 // Fast-path which gets the field from thread-local cache. 2006% fetch_from_thread_cache("%rax", miss_label="2f") 20071: 2008 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 2009 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 2010 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 2011 jne 3f 20125: 2013 POISON_HEAP_REF ebp 2014 movl %ebp, (%eax, %edx, 1) 2015 testl %ebp, %ebp 2016 je 4f 2017 movq rSELF:THREAD_CARD_TABLE_OFFSET, %rcx 2018 shrq $$CARD_TABLE_CARD_SHIFT, %rax 2019 movb %cl, (%rax, %rcx, 1) 20204: 2021 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 20222: 2023 EXPORT_PC 2024 movq rSELF:THREAD_SELF_OFFSET, %rdi 2025 movq 0(%rsp), %rsi 2026 movq rPC, %rdx 2027 movq %rbp, %rcx 2028 call nterp_get_static_field 2029 // Reload the value as it may have moved. 2030 GET_VREG %ebp, rINSTq 2031 testq MACRO_LITERAL(1), %rax 2032 je 1b 2033 CLEAR_VOLATILE_MARKER %rax 2034 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 2035 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 2036 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 2037 jne 7f 20386: 2039 POISON_HEAP_REF ebp 2040 movl %ebp, (%eax, %edx, 1) 2041 testl %ebp, %ebp 2042 je 8f 2043 movq rSELF:THREAD_CARD_TABLE_OFFSET, %rcx 2044 shrq $$CARD_TABLE_CARD_SHIFT, %rax 2045 movb %cl, (%rax, %rcx, 1) 20468: 2047 lock addl $$0, (%rsp) 2048 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 20493: 2050 call art_quick_read_barrier_mark_reg00 2051 jmp 5b 20527: 2053 call art_quick_read_barrier_mark_reg00 2054 jmp 6b 2055 2056NterpGetObjectStaticField: 2057 // Fast-path which gets the field from thread-local cache. 2058% fetch_from_thread_cache("%rax", miss_label="2f") 20591: 2060 movl ART_FIELD_OFFSET_OFFSET(%rax), %edx 2061 movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax 2062 cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET 2063 jne 5f 20646: 2065 testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%eax) 2066 movl (%eax, %edx, 1), %eax 2067 jnz 3f 2068 UNPOISON_HEAP_REF eax // Affects flags, so we cannot unpoison before the jnz. 20694: 2070 SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value 2071 ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 20722: 2073 EXPORT_PC 2074 movq rSELF:THREAD_SELF_OFFSET, %rdi 2075 movq 0(%rsp), %rsi 2076 movq rPC, %rdx 2077 movq $$0, %rcx 2078 call nterp_get_static_field 2079 andq $$-2, %rax 2080 jmp 1b 20813: 2082 UNPOISON_HEAP_REF eax 2083 call art_quick_read_barrier_mark_reg00 2084 jmp 4b 20855: 2086 call art_quick_read_barrier_mark_reg00 2087 jmp 6b 2088 2089NterpGetBooleanStaticField: 2090 OP_SGET load="movsbl", wide=0 2091 2092NterpGetByteStaticField: 2093 OP_SGET load="movsbl", wide=0 2094 2095NterpGetCharStaticField: 2096 OP_SGET load="movzwl", wide=0 2097 2098NterpGetShortStaticField: 2099 OP_SGET load="movswl", wide=0 2100 2101NterpGetWideStaticField: 2102 OP_SGET load="movq", wide=1 2103 2104NterpGetIntStaticField: 2105 OP_SGET load="movl", wide=0 2106 2107NterpPutStaticField: 2108 OP_SPUT rINST_reg=rINST, store="movl", wide=0 2109 2110NterpPutBooleanStaticField: 2111NterpPutByteStaticField: 2112 OP_SPUT rINST_reg=rINSTbl, store="movb", wide=0 2113 2114NterpPutCharStaticField: 2115NterpPutShortStaticField: 2116 OP_SPUT rINST_reg=rINSTw, store="movw", wide=0 2117 2118NterpPutWideStaticField: 2119 OP_SPUT rINST_reg=rINSTq, store="movq", wide=1 2120 2121NterpPutInstanceField: 2122 OP_IPUT rINST_reg=rINST, store="movl", wide=0 2123 2124NterpPutBooleanInstanceField: 2125NterpPutByteInstanceField: 2126 OP_IPUT rINST_reg=rINSTbl, store="movb", wide=0 2127 2128NterpPutCharInstanceField: 2129NterpPutShortInstanceField: 2130 OP_IPUT rINST_reg=rINSTw, store="movw", wide=0 2131 2132NterpPutWideInstanceField: 2133 OP_IPUT rINST_reg=rINSTq, store="movq", wide=1 2134 2135NterpGetBooleanInstanceField: 2136 OP_IGET load="movzbl", wide=0 2137 2138NterpGetByteInstanceField: 2139 OP_IGET load="movsbl", wide=0 2140 2141NterpGetCharInstanceField: 2142 OP_IGET load="movzwl", wide=0 2143 2144NterpGetShortInstanceField: 2145 OP_IGET load="movswl", wide=0 2146 2147NterpGetWideInstanceField: 2148 OP_IGET load="movq", wide=1 2149 2150NterpGetInstanceField: 2151 OP_IGET load="movl", wide=0 2152 2153NterpHandleHotnessOverflow: 2154 CHECK_AND_UPDATE_SHARED_MEMORY_METHOD if_hot=1f, if_not_hot=4f 21551: 2156 movq rPC, %rsi 2157 movq rFP, %rdx 2158 call nterp_hot_method 2159 testq %rax, %rax 2160 jne 3f 21612: 2162 FETCH_INST 2163 GOTO_NEXT 21643: 2165 // Drop the current frame. 2166 movq -8(rREFS), %rsp 2167 CFI_DEF_CFA(rsp, CALLEE_SAVES_SIZE) 2168 2169 // Setup the new frame 2170 movq OSR_DATA_FRAME_SIZE(%rax), %rcx 2171 // Given stack size contains all callee saved registers, remove them. 2172 subq $$CALLEE_SAVES_SIZE, %rcx 2173 2174 // Remember CFA. 2175 movq %rsp, %rbp 2176 CFI_DEF_CFA_REGISTER(rbp) 2177 2178 subq %rcx, %rsp 2179 movq %rsp, %rdi // rdi := beginning of stack 2180 leaq OSR_DATA_MEMORY(%rax), %rsi // rsi := memory to copy 2181 rep movsb // while (rcx--) { *rdi++ = *rsi++ } 2182 2183 // Fetch the native PC to jump to and save it in a callee-save register. 2184 movq OSR_DATA_NATIVE_PC(%rax), %rbx 2185 2186 // Free the memory holding OSR Data. 2187 movq %rax, %rdi 2188 call free 2189 2190 // Jump to the compiled code. 2191 jmp *%rbx 21924: 2193 DO_SUSPEND_CHECK continue_label=2b 2194 jmp 2b 2195 2196NterpHandleInvokeInterfaceOnObjectMethodRange: 2197 shrl $$16, %eax 2198 movq MIRROR_CLASS_VTABLE_OFFSET_64(%edx, %eax, 8), %rdi 2199 jmp NterpCommonInvokeInstanceRange 2200 2201NterpHandleInvokeInterfaceOnObjectMethod: 2202 shrl $$16, %eax 2203 movq MIRROR_CLASS_VTABLE_OFFSET_64(%edx, %eax, 8), %rdi 2204 jmp NterpCommonInvokeInstance 2205 2206// This is the logical end of ExecuteNterpImpl, where the frame info applies. 2207// EndExecuteNterpImpl includes the methods below as we want the runtime to 2208// see them as part of the Nterp PCs. 2209.cfi_endproc 2210 2211nterp_to_nterp_static_non_range: 2212 .cfi_startproc 2213 .cfi_def_cfa rsp, 8 2214 SETUP_STACK_FOR_INVOKE 2215 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=1, is_string_init=0 2216 .cfi_endproc 2217 2218nterp_to_nterp_string_init_non_range: 2219 .cfi_startproc 2220 .cfi_def_cfa rsp, 8 2221 SETUP_STACK_FOR_INVOKE 2222 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=1 2223 .cfi_endproc 2224 2225nterp_to_nterp_instance_non_range: 2226 .cfi_startproc 2227 .cfi_def_cfa rsp, 8 2228 SETUP_STACK_FOR_INVOKE 2229 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 2230 .cfi_endproc 2231 2232nterp_to_nterp_static_range: 2233 .cfi_startproc 2234 .cfi_def_cfa rsp, 8 2235 SETUP_STACK_FOR_INVOKE 2236 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=1 2237 .cfi_endproc 2238 2239nterp_to_nterp_instance_range: 2240 .cfi_startproc 2241 .cfi_def_cfa rsp, 8 2242 SETUP_STACK_FOR_INVOKE 2243 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0 2244 .cfi_endproc 2245 2246nterp_to_nterp_string_init_range: 2247 .cfi_startproc 2248 .cfi_def_cfa rsp, 8 2249 SETUP_STACK_FOR_INVOKE 2250 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=1 2251 .cfi_endproc 2252 2253END nterp_helper 2254 2255// This is the end of PCs contained by the OatQuickMethodHeader created for the interpreter 2256// entry point. 2257 FUNCTION_TYPE(EndExecuteNterpImpl) 2258 ASM_HIDDEN SYMBOL(EndExecuteNterpImpl) 2259 .global SYMBOL(EndExecuteNterpImpl) 2260SYMBOL(EndExecuteNterpImpl): 2261 2262// Entrypoints into runtime. 2263NTERP_TRAMPOLINE nterp_get_static_field, NterpGetStaticField 2264NTERP_TRAMPOLINE nterp_get_instance_field_offset, NterpGetInstanceFieldOffset 2265NTERP_TRAMPOLINE nterp_filled_new_array, NterpFilledNewArray 2266NTERP_TRAMPOLINE nterp_filled_new_array_range, NterpFilledNewArrayRange 2267NTERP_TRAMPOLINE nterp_get_class, NterpGetClass 2268NTERP_TRAMPOLINE nterp_allocate_object, NterpAllocateObject 2269NTERP_TRAMPOLINE nterp_get_method, NterpGetMethod 2270NTERP_TRAMPOLINE nterp_hot_method, NterpHotMethod 2271NTERP_TRAMPOLINE nterp_load_object, NterpLoadObject 2272 2273DEFINE_FUNCTION nterp_deliver_pending_exception 2274 DELIVER_PENDING_EXCEPTION 2275END_FUNCTION nterp_deliver_pending_exception 2276 2277// gen_mterp.py will inline the following definitions 2278// within [ExecuteNterpImpl, EndExecuteNterpImpl). 2279%def instruction_end(): 2280 2281 FUNCTION_TYPE(artNterpAsmInstructionEnd) 2282 ASM_HIDDEN SYMBOL(artNterpAsmInstructionEnd) 2283 .global SYMBOL(artNterpAsmInstructionEnd) 2284SYMBOL(artNterpAsmInstructionEnd): 2285 // artNterpAsmInstructionEnd is used as landing pad for exception handling. 2286 FETCH_INST 2287 GOTO_NEXT 2288 2289%def instruction_start(): 2290 2291 FUNCTION_TYPE(artNterpAsmInstructionStart) 2292 ASM_HIDDEN SYMBOL(artNterpAsmInstructionStart) 2293 .global SYMBOL(artNterpAsmInstructionStart) 2294SYMBOL(artNterpAsmInstructionStart) = .L_op_nop 2295 .text 2296 2297%def opcode_name_prefix(): 2298% return "nterp_" 2299%def opcode_start(): 2300 ENTRY nterp_${opcode} 2301%def opcode_end(): 2302 END nterp_${opcode} 2303 // Advance to the end of this handler. Causes error if we are past that point. 2304 .org nterp_${opcode} + NTERP_HANDLER_SIZE // ${opcode} handler is too big! 2305%def opcode_slow_path_start(name): 2306 ENTRY ${name} 2307%def opcode_slow_path_end(name): 2308 END ${name} 2309