< prev index next >

src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp

Print this page

  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.hpp"
  29 #include "asm/macroAssembler.inline.hpp"

  30 #include "code/codeCache.hpp"
  31 #include "code/debugInfoRec.hpp"
  32 #include "code/icBuffer.hpp"
  33 #include "code/vtableStubs.hpp"
  34 #include "compiler/oopMap.hpp"
  35 #include "gc/shared/barrierSetAssembler.hpp"
  36 #include "interpreter/interpreter.hpp"
  37 #include "interpreter/interp_masm.hpp"
  38 #include "logging/log.hpp"
  39 #include "memory/resourceArea.hpp"
  40 #include "nativeInst_aarch64.hpp"
  41 #include "oops/compiledICHolder.hpp"
  42 #include "oops/klass.inline.hpp"
  43 #include "prims/methodHandles.hpp"
  44 #include "runtime/jniHandles.hpp"
  45 #include "runtime/safepointMechanism.hpp"
  46 #include "runtime/sharedRuntime.hpp"
  47 #include "runtime/signature.hpp"
  48 #include "runtime/stubRoutines.hpp"
  49 #include "runtime/vframeArray.hpp"

 303     case T_SHORT:
 304     case T_INT:
 305       if (int_args < Argument::n_int_register_parameters_j) {
 306         regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
 307       } else {
 308         regs[i].set1(VMRegImpl::stack2reg(stk_args));
 309         stk_args += 2;
 310       }
 311       break;
 312     case T_VOID:
 313       // halves of T_LONG or T_DOUBLE
 314       assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
 315       regs[i].set_bad();
 316       break;
 317     case T_LONG:
 318       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 319       // fall through
 320     case T_OBJECT:
 321     case T_ARRAY:
 322     case T_ADDRESS:

 323       if (int_args < Argument::n_int_register_parameters_j) {
 324         regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
 325       } else {
 326         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 327         stk_args += 2;
 328       }
 329       break;
 330     case T_FLOAT:
 331       if (fp_args < Argument::n_float_register_parameters_j) {
 332         regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
 333       } else {
 334         regs[i].set1(VMRegImpl::stack2reg(stk_args));
 335         stk_args += 2;
 336       }
 337       break;
 338     case T_DOUBLE:
 339       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 340       if (fp_args < Argument::n_float_register_parameters_j) {
 341         regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
 342       } else {
 343         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 344         stk_args += 2;
 345       }
 346       break;
 347     default:
 348       ShouldNotReachHere();
 349       break;
 350     }
 351   }
 352 
 353   return align_up(stk_args, 2);
 354 }
 355 
















































































 356 // Patch the callers callsite with entry to compiled code if it exists.
 357 static void patch_callers_callsite(MacroAssembler *masm) {
 358   Label L;
 359   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 360   __ cbz(rscratch1, L);
 361 
 362   __ enter();
 363   __ push_CPU_state();
 364 
 365   // VM needs caller's callsite
 366   // VM needs target method
 367   // This needs to be a long call since we will relocate this adapter to
 368   // the codeBuffer and it may not reach
 369 
 370 #ifndef PRODUCT
 371   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
 372 #endif
 373 
 374   __ mov(c_rarg0, rmethod);
 375   __ mov(c_rarg1, lr);
 376   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
 377   __ blr(rscratch1);
 378 
 379   // Explicit isb required because fixup_callers_callsite may change the code
 380   // stream.
 381   __ safepoint_isb();
 382 
 383   __ pop_CPU_state();
 384   // restore sp
 385   __ leave();
 386   __ bind(L);
 387 }
 388 













































































































 389 static void gen_c2i_adapter(MacroAssembler *masm,
 390                             int total_args_passed,
 391                             int comp_args_on_stack,
 392                             const BasicType *sig_bt,
 393                             const VMRegPair *regs,
 394                             Label& skip_fixup) {






 395   // Before we get into the guts of the C2I adapter, see if we should be here
 396   // at all.  We've come from compiled code and are attempting to jump to the
 397   // interpreter, which means the caller made a static call to get here
 398   // (vcalls always get a compiled target if there is one).  Check for a
 399   // compiled target.  If there is one, we need to patch the caller's call.
 400   patch_callers_callsite(masm);
 401 
 402   __ bind(skip_fixup);
 403 
 404   int words_pushed = 0;





















 405 
 406   // Since all args are passed on the stack, total_args_passed *
 407   // Interpreter::stackElementSize is the space we need.
 408 
 409   int extraspace = total_args_passed * Interpreter::stackElementSize;

 410 
 411   __ mov(r13, sp);


 412 
 413   // stack is aligned, keep it that way
 414   extraspace = align_up(extraspace, 2*wordSize);

 415 
 416   if (extraspace)
 417     __ sub(sp, sp, extraspace);
 418 
 419   // Now write the args into the outgoing interpreter space
 420   for (int i = 0; i < total_args_passed; i++) {
 421     if (sig_bt[i] == T_VOID) {
 422       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
 423       continue;
 424     }
 425 
 426     // offset to start parameters
 427     int st_off   = (total_args_passed - i - 1) * Interpreter::stackElementSize;
 428     int next_off = st_off - Interpreter::stackElementSize;
 429 
 430     // Say 4 args:
 431     // i   st_off
 432     // 0   32 T_LONG
 433     // 1   24 T_VOID
 434     // 2   16 T_OBJECT
 435     // 3    8 T_BOOL
 436     // -    0 return address
 437     //
 438     // However to make thing extra confusing. Because we can fit a Java long/double in
 439     // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
 440     // leaves one slot empty and only stores to a single slot. In this case the
 441     // slot that is occupied is the T_VOID slot. See I said it was confusing.
 442 
 443     VMReg r_1 = regs[i].first();
 444     VMReg r_2 = regs[i].second();
 445     if (!r_1->is_valid()) {
 446       assert(!r_2->is_valid(), "");
 447       continue;
 448     }
 449     if (r_1->is_stack()) {
 450       // memory to memory use rscratch1
 451       int ld_off = (r_1->reg2stack() * VMRegImpl::stack_slot_size
 452                     + extraspace
 453                     + words_pushed * wordSize);
 454       if (!r_2->is_valid()) {
 455         // sign extend??
 456         __ ldrw(rscratch1, Address(sp, ld_off));
 457         __ str(rscratch1, Address(sp, st_off));
 458 
 459       } else {












 460 
 461         __ ldr(rscratch1, Address(sp, ld_off));
 462 
 463         // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
 464         // T_DOUBLE and T_LONG use two slots in the interpreter
 465         if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
 466           // ld_off == LSW, ld_off+wordSize == MSW
 467           // st_off == MSW, next_off == LSW
 468           __ str(rscratch1, Address(sp, next_off));


















 469 #ifdef ASSERT
 470           // Overwrite the unused slot with known junk
 471           __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaaaull);
 472           __ str(rscratch1, Address(sp, st_off));
 473 #endif /* ASSERT */
 474         } else {
 475           __ str(rscratch1, Address(sp, st_off));
 476         }
 477       }
 478     } else if (r_1->is_Register()) {
 479       Register r = r_1->as_Register();
 480       if (!r_2->is_valid()) {
 481         // must be only an int (or less ) so move only 32bits to slot
 482         // why not sign extend??
 483         __ str(r, Address(sp, st_off));
 484       } else {
 485         // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
 486         // T_DOUBLE and T_LONG use two slots in the interpreter
 487         if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
 488           // jlong/double in gpr
 489 #ifdef ASSERT
 490           // Overwrite the unused slot with known junk
 491           __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaabull);
 492           __ str(rscratch1, Address(sp, st_off));
 493 #endif /* ASSERT */
 494           __ str(r, Address(sp, next_off));






















 495         } else {
 496           __ str(r, Address(sp, st_off));





 497         }
 498       }
 499     } else {
 500       assert(r_1->is_FloatRegister(), "");
 501       if (!r_2->is_valid()) {
 502         // only a float use just part of the slot
 503         __ strs(r_1->as_FloatRegister(), Address(sp, st_off));
 504       } else {
 505 #ifdef ASSERT
 506         // Overwrite the unused slot with known junk
 507         __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaacull);
 508         __ str(rscratch1, Address(sp, st_off));
 509 #endif /* ASSERT */
 510         __ strd(r_1->as_FloatRegister(), Address(sp, next_off));
 511       }
 512     }
 513   }
 514 
 515   __ mov(esp, sp); // Interp expects args on caller's expression stack
 516 
 517   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));
 518   __ br(rscratch1);
 519 }
 520 

 521 
 522 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
 523                                     int total_args_passed,
 524                                     int comp_args_on_stack,
 525                                     const BasicType *sig_bt,
 526                                     const VMRegPair *regs) {
 527 
 528   // Note: r13 contains the senderSP on entry. We must preserve it since
 529   // we may do a i2c -> c2i transition if we lose a race where compiled
 530   // code goes non-entrant while we get args ready.
 531 
 532   // In addition we use r13 to locate all the interpreter args because
 533   // we must align the stack to 16 bytes.
 534 
 535   // Adapters are frameless.
 536 
 537   // An i2c adapter is frameless because the *caller* frame, which is
 538   // interpreted, routinely repairs its own esp (from
 539   // interpreter_frame_last_sp), even if a callee has modified the
 540   // stack pointer.  It also recalculates and aligns sp.
 541 
 542   // A c2i adapter is frameless because the *callee* frame, which is
 543   // interpreted, routinely repairs its caller's sp (from sender_sp,
 544   // which is set up via the senderSP register).
 545 
 546   // In other words, if *either* the caller or callee is interpreted, we can

 566       range_check(masm, rax, r11,
 567                   Interpreter::code()->code_start(), Interpreter::code()->code_end(),
 568                   L_ok);
 569     if (StubRoutines::code1() != NULL)
 570       range_check(masm, rax, r11,
 571                   StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
 572                   L_ok);
 573     if (StubRoutines::code2() != NULL)
 574       range_check(masm, rax, r11,
 575                   StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
 576                   L_ok);
 577     const char* msg = "i2c adapter must return to an interpreter frame";
 578     __ block_comment(msg);
 579     __ stop(msg);
 580     __ bind(L_ok);
 581     __ block_comment("} verify_i2ce ");
 582 #endif
 583   }
 584 
 585   // Cut-out for having no stack args.
 586   int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord;
 587   if (comp_args_on_stack) {
 588     __ sub(rscratch1, sp, comp_words_on_stack * wordSize);
 589     __ andr(sp, rscratch1, -16);

 590   }
 591 
 592   // Will jump to the compiled code just as if compiled code was doing it.
 593   // Pre-load the register-jump target early, to schedule it better.
 594   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::from_compiled_offset())));
 595 
 596 #if INCLUDE_JVMCI
 597   if (EnableJVMCI) {
 598     // check if this call should be routed towards a specific entry point
 599     __ ldr(rscratch2, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 600     Label no_alternative_target;
 601     __ cbz(rscratch2, no_alternative_target);
 602     __ mov(rscratch1, rscratch2);
 603     __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 604     __ bind(no_alternative_target);
 605   }
 606 #endif // INCLUDE_JVMCI
 607 


 608   // Now generate the shuffle code.
 609   for (int i = 0; i < total_args_passed; i++) {
 610     if (sig_bt[i] == T_VOID) {
 611       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");



 612       continue;
 613     }
 614 
 615     // Pick up 0, 1 or 2 words from SP+offset.

 616 
 617     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(),
 618             "scrambled load targets?");
 619     // Load in argument order going down.
 620     int ld_off = (total_args_passed - i - 1)*Interpreter::stackElementSize;
 621     // Point to interpreter value (vs. tag)
 622     int next_off = ld_off - Interpreter::stackElementSize;
 623     //
 624     //
 625     //
 626     VMReg r_1 = regs[i].first();
 627     VMReg r_2 = regs[i].second();
 628     if (!r_1->is_valid()) {
 629       assert(!r_2->is_valid(), "");
 630       continue;
 631     }
 632     if (r_1->is_stack()) {
 633       // Convert stack slot to an SP offset (+ wordSize to account for return address )
 634       int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size;
 635       if (!r_2->is_valid()) {
 636         // sign extend???
 637         __ ldrsw(rscratch2, Address(esp, ld_off));
 638         __ str(rscratch2, Address(sp, st_off));
 639       } else {
 640         //
 641         // We are using two optoregs. This can be either T_OBJECT,
 642         // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 643         // two slots but only uses one for thr T_LONG or T_DOUBLE case
 644         // So we must adjust where to pick up the data to match the
 645         // interpreter.
 646         //
 647         // Interpreter local[n] == MSW, local[n+1] == LSW however locals
 648         // are accessed as negative so LSW is at LOW address
 649 
 650         // ld_off is MSW so get LSW
 651         const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
 652                            next_off : ld_off;
 653         __ ldr(rscratch2, Address(esp, offset));
 654         // st_off is LSW (i.e. reg.first())
 655         __ str(rscratch2, Address(sp, st_off));
 656       }
 657     } else if (r_1->is_Register()) {  // Register argument
 658       Register r = r_1->as_Register();
 659       if (r_2->is_valid()) {
 660         //
 661         // We are using two VMRegs. This can be either T_OBJECT,
 662         // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 663         // two slots but only uses one for thr T_LONG or T_DOUBLE case
 664         // So we must adjust where to pick up the data to match the
 665         // interpreter.

















 666 
 667         const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
 668                            next_off : ld_off;
 669 
 670         // this can be a misaligned move
 671         __ ldr(r, Address(esp, offset));
 672       } else {
 673         // sign extend and use a full word?
 674         __ ldrw(r, Address(esp, ld_off));
 675       }
 676     } else {
 677       if (!r_2->is_valid()) {
 678         __ ldrs(r_1->as_FloatRegister(), Address(esp, ld_off));
 679       } else {
 680         __ ldrd(r_1->as_FloatRegister(), Address(esp, next_off));
 681       }
 682     }
 683   }
 684 
 685   // 6243940 We might end up in handle_wrong_method if
 686   // the callee is deoptimized as we race thru here. If that
 687   // happens we don't want to take a safepoint because the
 688   // caller frame will look interpreted and arguments are now
 689   // "compiled" so it is much better to make this transition
 690   // invisible to the stack walking code. Unfortunately if
 691   // we try and find the callee by normal means a safepoint
 692   // is possible. So we stash the desired callee in the thread
 693   // and the vm will find there should this case occur.
 694 
 695   __ str(rmethod, Address(rthread, JavaThread::callee_target_offset()));
 696 
 697   __ br(rscratch1);
 698 }
 699 
 700 // ---------------------------------------------------------------
 701 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
 702                                                             int total_args_passed,
 703                                                             int comp_args_on_stack,
 704                                                             const BasicType *sig_bt,
 705                                                             const VMRegPair *regs,
 706                                                             AdapterFingerPrint* fingerprint) {
 707   address i2c_entry = __ pc();
 708 
 709   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
 710 
 711   address c2i_unverified_entry = __ pc();
 712   Label skip_fixup;
 713 
 714   Label ok;
 715 
 716   Register holder = rscratch2;
 717   Register receiver = j_rarg0;
 718   Register tmp = r10;  // A call-clobbered register not used for arg passing
 719 
 720   // -------------------------------------------------------------------------
 721   // Generate a C2I adapter.  On entry we know rmethod holds the Method* during calls
 722   // to the interpreter.  The args start out packed in the compiled layout.  They
 723   // need to be unpacked into the interpreter layout.  This will almost always
 724   // require some stack space.  We grow the current (compiled) stack, then repack
 725   // the args.  We  finally end in a jump to the generic interpreter entry point.
 726   // On exit from the interpreter, the interpreter will restore our SP (lest the
 727   // compiled code, which relys solely on SP and not FP, get sick).
 728 
 729   {
 730     __ block_comment("c2i_unverified_entry {");
 731     __ load_klass(rscratch1, receiver);
 732     __ ldr(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
 733     __ cmp(rscratch1, tmp);
 734     __ ldr(rmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
 735     __ br(Assembler::EQ, ok);
 736     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 737 
 738     __ bind(ok);
 739     // Method might have been compiled since the call site was patched to
 740     // interpreted; if that is the case treat it as a miss so we can get
 741     // the call site corrected.
 742     __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 743     __ cbz(rscratch1, skip_fixup);
 744     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 745     __ block_comment("} c2i_unverified_entry");
 746   }


































 747 

 748   address c2i_entry = __ pc();
 749 
 750   // Class initialization barrier for static methods
 751   address c2i_no_clinit_check_entry = NULL;
 752   if (VM_Version::supports_fast_class_init_checks()) {
 753     Label L_skip_barrier;
 754 
 755     { // Bypass the barrier for non-static methods
 756       __ ldrw(rscratch1, Address(rmethod, Method::access_flags_offset()));
 757       __ andsw(zr, rscratch1, JVM_ACC_STATIC);
 758       __ br(Assembler::EQ, L_skip_barrier); // non-static
 759     }
 760 
 761     __ load_method_holder(rscratch2, rmethod);
 762     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 763     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 764 
 765     __ bind(L_skip_barrier);
 766     c2i_no_clinit_check_entry = __ pc();
 767   }
 768 
 769   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 770   bs->c2i_entry_barrier(masm);
 771 
 772   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);













 773 
 774   __ flush();
 775   return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);








 776 }
 777 
 778 static int c_calling_convention_priv(const BasicType *sig_bt,
 779                                          VMRegPair *regs,
 780                                          VMRegPair *regs2,
 781                                          int total_args_passed) {
 782   assert(regs2 == NULL, "not needed on AArch64");
 783 
 784 // We return the amount of VMRegImpl stack slots we need to reserve for all
 785 // the arguments NOT counting out_preserve_stack_slots.
 786 
 787     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
 788       c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5,  c_rarg6,  c_rarg7
 789     };
 790     static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
 791       c_farg0, c_farg1, c_farg2, c_farg3,
 792       c_farg4, c_farg5, c_farg6, c_farg7
 793     };
 794 
 795     uint int_args = 0;

 803       case T_BYTE:
 804       case T_SHORT:
 805       case T_INT:
 806         if (int_args < Argument::n_int_register_parameters_c) {
 807           regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
 808         } else {
 809 #ifdef __APPLE__
 810           // Less-than word types are stored one after another.
 811           // The code is unable to handle this so bailout.
 812           return -1;
 813 #endif
 814           regs[i].set1(VMRegImpl::stack2reg(stk_args));
 815           stk_args += 2;
 816         }
 817         break;
 818       case T_LONG:
 819         assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 820         // fall through
 821       case T_OBJECT:
 822       case T_ARRAY:

 823       case T_ADDRESS:
 824       case T_METADATA:
 825         if (int_args < Argument::n_int_register_parameters_c) {
 826           regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
 827         } else {
 828           regs[i].set2(VMRegImpl::stack2reg(stk_args));
 829           stk_args += 2;
 830         }
 831         break;
 832       case T_FLOAT:
 833         if (fp_args < Argument::n_float_register_parameters_c) {
 834           regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
 835         } else {
 836 #ifdef __APPLE__
 837           // Less-than word types are stored one after another.
 838           // The code is unable to handle this so bailout.
 839           return -1;
 840 #endif
 841           regs[i].set1(VMRegImpl::stack2reg(stk_args));
 842           stk_args += 2;

1643       reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1644     } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1645       freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1646     }
1647 #endif /* ASSERT */
1648     switch (in_sig_bt[i]) {
1649       case T_ARRAY:
1650         if (is_critical_native) {
1651           unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]);
1652           c_arg++;
1653 #ifdef ASSERT
1654           if (out_regs[c_arg].first()->is_Register()) {
1655             reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1656           } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1657             freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1658           }
1659 #endif
1660           int_args++;
1661           break;
1662         }

1663       case T_OBJECT:
1664         assert(!is_critical_native, "no oop arguments");
1665         object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
1666                     ((i == 0) && (!is_static)),
1667                     &receiver_offset);
1668         int_args++;
1669         break;
1670       case T_VOID:
1671         break;
1672 
1673       case T_FLOAT:
1674         float_move(masm, in_regs[i], out_regs[c_arg]);
1675         float_args++;
1676         break;
1677 
1678       case T_DOUBLE:
1679         assert( i + 1 < total_in_args &&
1680                 in_sig_bt[i + 1] == T_VOID &&
1681                 out_sig_bt[c_arg+1] == T_VOID, "bad arg list");
1682         double_move(masm, in_regs[i], out_regs[c_arg]);

1759   Label lock_done;
1760 
1761   if (method->is_synchronized()) {
1762     assert(!is_critical_native, "unhandled");
1763 
1764     const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
1765 
1766     // Get the handle (the 2nd argument)
1767     __ mov(oop_handle_reg, c_rarg1);
1768 
1769     // Get address of the box
1770 
1771     __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
1772 
1773     // Load the oop from the handle
1774     __ ldr(obj_reg, Address(oop_handle_reg, 0));
1775 
1776     // Load (object->mark() | 1) into swap_reg %r0
1777     __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1778     __ orr(swap_reg, rscratch1, 1);




1779 
1780     // Save (object->mark() | 1) into BasicLock's displaced header
1781     __ str(swap_reg, Address(lock_reg, mark_word_offset));
1782 
1783     // src -> dest iff dest == r0 else r0 <- dest
1784     { Label here;
1785       __ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, lock_done, /*fallthrough*/NULL);
1786     }
1787 
1788     // Hmm should this move to the slow path code area???
1789 
1790     // Test if the oopMark is an obvious stack pointer, i.e.,
1791     //  1) (mark & 3) == 0, and
1792     //  2) sp <= mark < mark + os::pagesize()
1793     // These 3 tests can be done by evaluating the following
1794     // expression: ((mark - sp) & (3 - os::vm_page_size())),
1795     // assuming both stack pointer and pagesize have their
1796     // least significant 2 bits clear.
1797     // NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg
1798 

1824 
1825   rt_call(masm, native_func);
1826 
1827   __ bind(native_return);
1828 
1829   intptr_t return_pc = (intptr_t) __ pc();
1830   oop_maps->add_gc_map(return_pc - start, map);
1831 
1832   // Unpack native results.
1833   switch (ret_type) {
1834   case T_BOOLEAN: __ c2bool(r0);                     break;
1835   case T_CHAR   : __ ubfx(r0, r0, 0, 16);            break;
1836   case T_BYTE   : __ sbfx(r0, r0, 0, 8);             break;
1837   case T_SHORT  : __ sbfx(r0, r0, 0, 16);            break;
1838   case T_INT    : __ sbfx(r0, r0, 0, 32);            break;
1839   case T_DOUBLE :
1840   case T_FLOAT  :
1841     // Result is in v0 we'll save as needed
1842     break;
1843   case T_ARRAY:                 // Really a handle

1844   case T_OBJECT:                // Really a handle
1845       break; // can't de-handlize until after safepoint check
1846   case T_VOID: break;
1847   case T_LONG: break;
1848   default       : ShouldNotReachHere();
1849   }
1850 
1851   Label safepoint_in_progress, safepoint_in_progress_done;
1852   Label after_transition;
1853 
1854   // If this is a critical native, check for a safepoint or suspend request after the call.
1855   // If a safepoint is needed, transition to native, then to native_trans to handle
1856   // safepoints like the native methods that are not critical natives.
1857   if (is_critical_native) {
1858     Label needs_safepoint;
1859     __ safepoint_poll(needs_safepoint, false /* at_return */, true /* acquire */, false /* in_nmethod */);
1860     __ ldrw(rscratch1, Address(rthread, JavaThread::suspend_flags_offset()));
1861     __ cbnzw(rscratch1, needs_safepoint);
1862     __ b(after_transition);
1863     __ bind(needs_safepoint);

3066 #ifdef ASSERT
3067   __ str(zr, Address(rthread, JavaThread::exception_handler_pc_offset()));
3068   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
3069 #endif
3070   // Clear the exception oop so GC no longer processes it as a root.
3071   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
3072 
3073   // r0: exception oop
3074   // r8:  exception handler
3075   // r4: exception pc
3076   // Jump to handler
3077 
3078   __ br(r8);
3079 
3080   // Make sure all code is generated
3081   masm->flush();
3082 
3083   // Set exception blob
3084   _exception_blob =  ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
3085 }


























































































































3086 
3087 // ---------------------------------------------------------------
3088 
3089 class NativeInvokerGenerator : public StubCodeGenerator {
3090   address _call_target;
3091   int _shadow_space_bytes;
3092 
3093   const GrowableArray<VMReg>& _input_registers;
3094   const GrowableArray<VMReg>& _output_registers;
3095 
3096   int _frame_complete;
3097   int _framesize;
3098   OopMapSet* _oop_maps;
3099 public:
3100   NativeInvokerGenerator(CodeBuffer* buffer,
3101                          address call_target,
3102                          int shadow_space_bytes,
3103                          const GrowableArray<VMReg>& input_registers,
3104                          const GrowableArray<VMReg>& output_registers)
3105    : StubCodeGenerator(buffer, PrintMethodHandleStubs),

3318 
3319   //////////////////////////////////////////////////////////////////////////////
3320 
3321   __ block_comment("{ L_reguard");
3322   __ bind(L_reguard);
3323 
3324   spill_output_registers();
3325 
3326   rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
3327 
3328   fill_output_registers();
3329 
3330   __ b(L_after_reguard);
3331 
3332   __ block_comment("} L_reguard");
3333 
3334   //////////////////////////////////////////////////////////////////////////////
3335 
3336   __ flush();
3337 }
3338 #endif // COMPILER2

  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.hpp"
  29 #include "asm/macroAssembler.inline.hpp"
  30 #include "classfile/symbolTable.hpp"
  31 #include "code/codeCache.hpp"
  32 #include "code/debugInfoRec.hpp"
  33 #include "code/icBuffer.hpp"
  34 #include "code/vtableStubs.hpp"
  35 #include "compiler/oopMap.hpp"
  36 #include "gc/shared/barrierSetAssembler.hpp"
  37 #include "interpreter/interpreter.hpp"
  38 #include "interpreter/interp_masm.hpp"
  39 #include "logging/log.hpp"
  40 #include "memory/resourceArea.hpp"
  41 #include "nativeInst_aarch64.hpp"
  42 #include "oops/compiledICHolder.hpp"
  43 #include "oops/klass.inline.hpp"
  44 #include "prims/methodHandles.hpp"
  45 #include "runtime/jniHandles.hpp"
  46 #include "runtime/safepointMechanism.hpp"
  47 #include "runtime/sharedRuntime.hpp"
  48 #include "runtime/signature.hpp"
  49 #include "runtime/stubRoutines.hpp"
  50 #include "runtime/vframeArray.hpp"

 304     case T_SHORT:
 305     case T_INT:
 306       if (int_args < Argument::n_int_register_parameters_j) {
 307         regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
 308       } else {
 309         regs[i].set1(VMRegImpl::stack2reg(stk_args));
 310         stk_args += 2;
 311       }
 312       break;
 313     case T_VOID:
 314       // halves of T_LONG or T_DOUBLE
 315       assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
 316       regs[i].set_bad();
 317       break;
 318     case T_LONG:
 319       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 320       // fall through
 321     case T_OBJECT:
 322     case T_ARRAY:
 323     case T_ADDRESS:
 324     case T_INLINE_TYPE:
 325       if (int_args < Argument::n_int_register_parameters_j) {
 326         regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
 327       } else {
 328         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 329         stk_args += 2;
 330       }
 331       break;
 332     case T_FLOAT:
 333       if (fp_args < Argument::n_float_register_parameters_j) {
 334         regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
 335       } else {
 336         regs[i].set1(VMRegImpl::stack2reg(stk_args));
 337         stk_args += 2;
 338       }
 339       break;
 340     case T_DOUBLE:
 341       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 342       if (fp_args < Argument::n_float_register_parameters_j) {
 343         regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
 344       } else {
 345         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 346         stk_args += 2;
 347       }
 348       break;
 349     default:
 350       ShouldNotReachHere();
 351       break;
 352     }
 353   }
 354 
 355   return align_up(stk_args, 2);
 356 }
 357 
 358 
 359 const uint SharedRuntime::java_return_convention_max_int = Argument::n_int_register_parameters_j;
 360 const uint SharedRuntime::java_return_convention_max_float = Argument::n_float_register_parameters_j;
 361 
 362 int SharedRuntime::java_return_convention(const BasicType *sig_bt, VMRegPair *regs, int total_args_passed) {
 363 
 364   // Create the mapping between argument positions and registers.
 365 
 366   static const Register INT_ArgReg[java_return_convention_max_int] = {
 367     r0 /* j_rarg7 */, j_rarg6, j_rarg5, j_rarg4, j_rarg3, j_rarg2, j_rarg1, j_rarg0
 368   };
 369 
 370   static const FloatRegister FP_ArgReg[java_return_convention_max_float] = {
 371     j_farg0, j_farg1, j_farg2, j_farg3, j_farg4, j_farg5, j_farg6, j_farg7
 372   };
 373 
 374   uint int_args = 0;
 375   uint fp_args = 0;
 376 
 377   for (int i = 0; i < total_args_passed; i++) {
 378     switch (sig_bt[i]) {
 379     case T_BOOLEAN:
 380     case T_CHAR:
 381     case T_BYTE:
 382     case T_SHORT:
 383     case T_INT:
 384       if (int_args < SharedRuntime::java_return_convention_max_int) {
 385         regs[i].set1(INT_ArgReg[int_args]->as_VMReg());
 386         int_args ++;
 387       } else {
 388         return -1;
 389       }
 390       break;
 391     case T_VOID:
 392       // halves of T_LONG or T_DOUBLE
 393       assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
 394       regs[i].set_bad();
 395       break;
 396     case T_LONG:
 397       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 398       // fall through
 399     case T_OBJECT:
 400     case T_ARRAY:
 401     case T_ADDRESS:
 402       // Should T_METADATA be added to java_calling_convention as well ?
 403     case T_METADATA:
 404     case T_INLINE_TYPE:
 405       if (int_args < SharedRuntime::java_return_convention_max_int) {
 406         regs[i].set2(INT_ArgReg[int_args]->as_VMReg());
 407         int_args ++;
 408       } else {
 409         return -1;
 410       }
 411       break;
 412     case T_FLOAT:
 413       if (fp_args < SharedRuntime::java_return_convention_max_float) {
 414         regs[i].set1(FP_ArgReg[fp_args]->as_VMReg());
 415         fp_args ++;
 416       } else {
 417         return -1;
 418       }
 419       break;
 420     case T_DOUBLE:
 421       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 422       if (fp_args < SharedRuntime::java_return_convention_max_float) {
 423         regs[i].set2(FP_ArgReg[fp_args]->as_VMReg());
 424         fp_args ++;
 425       } else {
 426         return -1;
 427       }
 428       break;
 429     default:
 430       ShouldNotReachHere();
 431       break;
 432     }
 433   }
 434 
 435   return int_args + fp_args;
 436 }
 437 
 438 // Patch the callers callsite with entry to compiled code if it exists.
 439 static void patch_callers_callsite(MacroAssembler *masm) {
 440   Label L;
 441   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 442   __ cbz(rscratch1, L);
 443 
 444   __ enter();
 445   __ push_CPU_state();
 446 
 447   // VM needs caller's callsite
 448   // VM needs target method
 449   // This needs to be a long call since we will relocate this adapter to
 450   // the codeBuffer and it may not reach
 451 
 452 #ifndef PRODUCT
 453   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
 454 #endif
 455 
 456   __ mov(c_rarg0, rmethod);
 457   __ mov(c_rarg1, lr);
 458   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
 459   __ blr(rscratch1);
 460 
 461   // Explicit isb required because fixup_callers_callsite may change the code
 462   // stream.
 463   __ safepoint_isb();
 464 
 465   __ pop_CPU_state();
 466   // restore sp
 467   __ leave();
 468   __ bind(L);
 469 }
 470 
 471 // For each inline type argument, sig includes the list of fields of
 472 // the inline type. This utility function computes the number of
 473 // arguments for the call if inline types are passed by reference (the
 474 // calling convention the interpreter expects).
 475 static int compute_total_args_passed_int(const GrowableArray<SigEntry>* sig_extended) {
 476   int total_args_passed = 0;
 477   if (InlineTypePassFieldsAsArgs) {
 478      for (int i = 0; i < sig_extended->length(); i++) {
 479        BasicType bt = sig_extended->at(i)._bt;
 480        if (bt == T_INLINE_TYPE) {
 481          // In sig_extended, an inline type argument starts with:
 482          // T_INLINE_TYPE, followed by the types of the fields of the
 483          // inline type and T_VOID to mark the end of the value
 484          // type. Inline types are flattened so, for instance, in the
 485          // case of an inline type with an int field and an inline type
 486          // field that itself has 2 fields, an int and a long:
 487          // T_INLINE_TYPE T_INT T_INLINE_TYPE T_INT T_LONG T_VOID (second
 488          // slot for the T_LONG) T_VOID (inner T_INLINE_TYPE) T_VOID
 489          // (outer T_INLINE_TYPE)
 490          total_args_passed++;
 491          int vt = 1;
 492          do {
 493            i++;
 494            BasicType bt = sig_extended->at(i)._bt;
 495            BasicType prev_bt = sig_extended->at(i-1)._bt;
 496            if (bt == T_INLINE_TYPE) {
 497              vt++;
 498            } else if (bt == T_VOID &&
 499                       prev_bt != T_LONG &&
 500                       prev_bt != T_DOUBLE) {
 501              vt--;
 502            }
 503          } while (vt != 0);
 504        } else {
 505          total_args_passed++;
 506        }
 507      }
 508   } else {
 509     total_args_passed = sig_extended->length();
 510   }
 511 
 512   return total_args_passed;
 513 }
 514 
 515 
 516 static void gen_c2i_adapter_helper(MacroAssembler* masm,
 517                                    BasicType bt,
 518                                    BasicType prev_bt,
 519                                    size_t size_in_bytes,
 520                                    const VMRegPair& reg_pair,
 521                                    const Address& to,
 522                                    Register tmp1,
 523                                    Register tmp2,
 524                                    Register tmp3,
 525                                    int extraspace,
 526                                    bool is_oop) {
 527   assert(bt != T_INLINE_TYPE || !InlineTypePassFieldsAsArgs, "no inline type here");
 528   if (bt == T_VOID) {
 529     assert(prev_bt == T_LONG || prev_bt == T_DOUBLE, "missing half");
 530     return;
 531   }
 532 
 533   // Say 4 args:
 534   // i   st_off
 535   // 0   32 T_LONG
 536   // 1   24 T_VOID
 537   // 2   16 T_OBJECT
 538   // 3    8 T_BOOL
 539   // -    0 return address
 540   //
 541   // However to make thing extra confusing. Because we can fit a Java long/double in
 542   // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
 543   // leaves one slot empty and only stores to a single slot. In this case the
 544   // slot that is occupied is the T_VOID slot. See I said it was confusing.
 545 
 546   bool wide = (size_in_bytes == wordSize);
 547   VMReg r_1 = reg_pair.first();
 548   VMReg r_2 = reg_pair.second();
 549   assert(r_2->is_valid() == wide, "invalid size");
 550   if (!r_1->is_valid()) {
 551     assert(!r_2->is_valid(), "");
 552     return;
 553   }
 554 
 555   if (!r_1->is_FloatRegister()) {
 556     Register val = tmp3;
 557     if (r_1->is_stack()) {
 558       // memory to memory use tmp3 (scratch registers are used by store_heap_oop)
 559       int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
 560       __ load_sized_value(val, Address(sp, ld_off), size_in_bytes, /* is_signed */ false);
 561     } else {
 562       val = r_1->as_Register();
 563     }
 564     assert_different_registers(to.base(), val, rscratch2, tmp1, tmp2);
 565     if (is_oop) {
 566       __ store_heap_oop(to, val, rscratch2, tmp1, tmp2, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
 567     } else {
 568       __ store_sized_value(to, val, size_in_bytes);
 569     }
 570   } else {
 571     if (wide) {
 572       __ strd(r_1->as_FloatRegister(), to);
 573     } else {
 574       // only a float use just part of the slot
 575       __ strs(r_1->as_FloatRegister(), to);
 576     }
 577   }
 578 }
 579 
 580 static void gen_c2i_adapter(MacroAssembler *masm,
 581                             const GrowableArray<SigEntry>* sig_extended,


 582                             const VMRegPair *regs,
 583                             Label& skip_fixup,
 584                             address start,
 585                             OopMapSet* oop_maps,
 586                             int& frame_complete,
 587                             int& frame_size_in_words,
 588                             bool alloc_inline_receiver) {
 589 
 590   // Before we get into the guts of the C2I adapter, see if we should be here
 591   // at all.  We've come from compiled code and are attempting to jump to the
 592   // interpreter, which means the caller made a static call to get here
 593   // (vcalls always get a compiled target if there is one).  Check for a
 594   // compiled target.  If there is one, we need to patch the caller's call.
 595   patch_callers_callsite(masm);
 596 
 597   __ bind(skip_fixup);
 598 
 599   // Name some registers to be used in the following code. We can use
 600   // anything except r0-r7 which are arguments in the Java calling
 601   // convention, rmethod (r12), and r13 which holds the outgoing sender
 602   // SP for the interpreter.
 603   Register buf_array = r10;   // Array of buffered inline types
 604   Register buf_oop = r11;     // Buffered inline type oop
 605   Register tmp1 = r15;
 606   Register tmp2 = r16;
 607   Register tmp3 = r17;
 608 
 609   if (InlineTypePassFieldsAsArgs) {
 610     // Is there an inline type argument?
 611     bool has_inline_argument = false;
 612     for (int i = 0; i < sig_extended->length() && !has_inline_argument; i++) {
 613       has_inline_argument = (sig_extended->at(i)._bt == T_INLINE_TYPE);
 614     }
 615     if (has_inline_argument) {
 616       // There is at least an inline type argument: we're coming from
 617       // compiled code so we have no buffers to back the inline types
 618       // Allocate the buffers here with a runtime call.
 619       RegisterSaver reg_save(false /* save_vectors */);
 620       OopMap* map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
 621 
 622       frame_complete = __ offset();
 623       address the_pc = __ pc();
 624 
 625       Label retaddr;
 626       __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
 627 
 628       __ mov(c_rarg0, rthread);
 629       __ mov(c_rarg1, rmethod);
 630       __ mov(c_rarg2, (int64_t)alloc_inline_receiver);
 631 
 632       __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::allocate_inline_types)));
 633       __ blr(rscratch1);
 634       __ bind(retaddr);
 635 
 636       oop_maps->add_gc_map(__ pc() - start, map);
 637       __ reset_last_Java_frame(false);
 638 
 639       reg_save.restore_live_registers(masm);





 640 
 641       Label no_exception;
 642       __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 643       __ cbz(rscratch1, no_exception);
 644 
 645       __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
 646       __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
 647       __ b(RuntimeAddress(StubRoutines::forward_exception_entry()));









 648 
 649       __ bind(no_exception);
 650 
 651       // We get an array of objects from the runtime call
 652       __ get_vm_result(buf_array, rthread);
 653       __ get_vm_result_2(rmethod, rthread); // TODO: required to keep the callee Method live?
 654     }
 655   }








 656 
 657   // Since all args are passed on the stack, total_args_passed *
 658   // Interpreter::stackElementSize is the space we need.
 659 
 660   int total_args_passed = compute_total_args_passed_int(sig_extended);
 661   int extraspace = total_args_passed * Interpreter::stackElementSize;
 662 
 663   // stack is aligned, keep it that way
 664   extraspace = align_up(extraspace, StackAlignmentInBytes);
 665 
 666   // set senderSP value
 667   __ mov(r13, sp);
 668 
 669   __ sub(sp, sp, extraspace);
 670 
 671   // Now write the args into the outgoing interpreter space
 672 
 673   // next_arg_comp is the next argument from the compiler point of
 674   // view (inline type fields are passed in registers/on the stack). In
 675   // sig_extended, an inline type argument starts with: T_INLINE_TYPE,
 676   // followed by the types of the fields of the inline type and T_VOID
 677   // to mark the end of the inline type. ignored counts the number of
 678   // T_INLINE_TYPE/T_VOID. next_vt_arg is the next inline type argument:
 679   // used to get the buffer for that argument from the pool of buffers
 680   // we allocated above and want to pass to the
 681   // interpreter. next_arg_int is the next argument from the
 682   // interpreter point of view (inline types are passed by reference).
 683   for (int next_arg_comp = 0, ignored = 0, next_vt_arg = 0, next_arg_int = 0;
 684        next_arg_comp < sig_extended->length(); next_arg_comp++) {
 685     assert(ignored <= next_arg_comp, "shouldn't skip over more slots than there are arguments");
 686     assert(next_arg_int <= total_args_passed, "more arguments for the interpreter than expected?");
 687     BasicType bt = sig_extended->at(next_arg_comp)._bt;
 688     int st_off = (total_args_passed - next_arg_int - 1) * Interpreter::stackElementSize;
 689     if (!InlineTypePassFieldsAsArgs || bt != T_INLINE_TYPE) {
 690       int next_off = st_off - Interpreter::stackElementSize;
 691       const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : st_off;
 692       const VMRegPair reg_pair = regs[next_arg_comp-ignored];
 693       size_t size_in_bytes = reg_pair.second()->is_valid() ? 8 : 4;
 694       gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
 695                              size_in_bytes, reg_pair, Address(sp, offset), tmp1, tmp2, tmp3, extraspace, false);
 696       next_arg_int++;
 697 #ifdef ASSERT
 698       if (bt == T_LONG || bt == T_DOUBLE) {
 699         // Overwrite the unused slot with known junk
 700         __ mov(rscratch1, CONST64(0xdeadffffdeadaaaa));
 701         __ str(rscratch1, Address(sp, st_off));



 702       }















 703 #endif /* ASSERT */
 704     } else {
 705       ignored++;
 706       // get the buffer from the just allocated pool of buffers
 707       int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + next_vt_arg * type2aelembytes(T_INLINE_TYPE);
 708       __ load_heap_oop(buf_oop, Address(buf_array, index));
 709       next_vt_arg++; next_arg_int++;
 710       int vt = 1;
 711       // write fields we get from compiled code in registers/stack
 712       // slots to the buffer: we know we are done with that inline type
 713       // argument when we hit the T_VOID that acts as an end of inline
 714       // type delimiter for this inline type. Inline types are flattened
 715       // so we might encounter embedded inline types. Each entry in
 716       // sig_extended contains a field offset in the buffer.
 717       do {
 718         next_arg_comp++;
 719         BasicType bt = sig_extended->at(next_arg_comp)._bt;
 720         BasicType prev_bt = sig_extended->at(next_arg_comp - 1)._bt;
 721         if (bt == T_INLINE_TYPE) {
 722           vt++;
 723           ignored++;
 724         } else if (bt == T_VOID && prev_bt != T_LONG && prev_bt != T_DOUBLE) {
 725           vt--;
 726           ignored++;
 727         } else {
 728           int off = sig_extended->at(next_arg_comp)._offset;
 729           assert(off > 0, "offset in object should be positive");
 730           size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
 731           bool is_oop = is_reference_type(bt);
 732           gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
 733                                  size_in_bytes, regs[next_arg_comp-ignored], Address(buf_oop, off), tmp1, tmp2, tmp3, extraspace, is_oop);
 734         }
 735       } while (vt != 0);
 736       // pass the buffer to the interpreter
 737       __ str(buf_oop, Address(sp, st_off));











 738     }
 739   }
 740 
 741   __ mov(esp, sp); // Interp expects args on caller's expression stack
 742 
 743   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));
 744   __ br(rscratch1);
 745 }
 746 
 747 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, int comp_args_on_stack, const GrowableArray<SigEntry>* sig, const VMRegPair *regs) {
 748 





 749 
 750   // Note: r13 contains the senderSP on entry. We must preserve it since
 751   // we may do a i2c -> c2i transition if we lose a race where compiled
 752   // code goes non-entrant while we get args ready.
 753 
 754   // In addition we use r13 to locate all the interpreter args because
 755   // we must align the stack to 16 bytes.
 756 
 757   // Adapters are frameless.
 758 
 759   // An i2c adapter is frameless because the *caller* frame, which is
 760   // interpreted, routinely repairs its own esp (from
 761   // interpreter_frame_last_sp), even if a callee has modified the
 762   // stack pointer.  It also recalculates and aligns sp.
 763 
 764   // A c2i adapter is frameless because the *callee* frame, which is
 765   // interpreted, routinely repairs its caller's sp (from sender_sp,
 766   // which is set up via the senderSP register).
 767 
 768   // In other words, if *either* the caller or callee is interpreted, we can

 788       range_check(masm, rax, r11,
 789                   Interpreter::code()->code_start(), Interpreter::code()->code_end(),
 790                   L_ok);
 791     if (StubRoutines::code1() != NULL)
 792       range_check(masm, rax, r11,
 793                   StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
 794                   L_ok);
 795     if (StubRoutines::code2() != NULL)
 796       range_check(masm, rax, r11,
 797                   StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
 798                   L_ok);
 799     const char* msg = "i2c adapter must return to an interpreter frame";
 800     __ block_comment(msg);
 801     __ stop(msg);
 802     __ bind(L_ok);
 803     __ block_comment("} verify_i2ce ");
 804 #endif
 805   }
 806 
 807   // Cut-out for having no stack args.
 808   int comp_words_on_stack = 0;
 809   if (comp_args_on_stack) {
 810      comp_words_on_stack = align_up(comp_args_on_stack * VMRegImpl::stack_slot_size, wordSize) >> LogBytesPerWord;
 811      __ sub(rscratch1, sp, comp_words_on_stack * wordSize);
 812      __ andr(sp, rscratch1, -16);
 813   }
 814 
 815   // Will jump to the compiled code just as if compiled code was doing it.
 816   // Pre-load the register-jump target early, to schedule it better.
 817   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::from_compiled_inline_offset())));
 818 
 819 #if INCLUDE_JVMCI
 820   if (EnableJVMCI) {
 821     // check if this call should be routed towards a specific entry point
 822     __ ldr(rscratch2, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 823     Label no_alternative_target;
 824     __ cbz(rscratch2, no_alternative_target);
 825     __ mov(rscratch1, rscratch2);
 826     __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 827     __ bind(no_alternative_target);
 828   }
 829 #endif // INCLUDE_JVMCI
 830 
 831   int total_args_passed = sig->length();
 832 
 833   // Now generate the shuffle code.
 834   for (int i = 0; i < total_args_passed; i++) {
 835     BasicType bt = sig->at(i)._bt;
 836 
 837     assert(bt != T_INLINE_TYPE, "i2c adapter doesn't unpack inline typ args");
 838     if (bt == T_VOID) {
 839       assert(i > 0 && (sig->at(i - 1)._bt == T_LONG || sig->at(i - 1)._bt == T_DOUBLE), "missing half");
 840       continue;
 841     }
 842 
 843     // Pick up 0, 1 or 2 words from SP+offset.
 844     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(), "scrambled load targets?");
 845 


 846     // Load in argument order going down.
 847     int ld_off = (total_args_passed - i - 1) * Interpreter::stackElementSize;
 848     // Point to interpreter value (vs. tag)
 849     int next_off = ld_off - Interpreter::stackElementSize;
 850     //
 851     //
 852     //
 853     VMReg r_1 = regs[i].first();
 854     VMReg r_2 = regs[i].second();
 855     if (!r_1->is_valid()) {
 856       assert(!r_2->is_valid(), "");
 857       continue;
 858     }
 859     if (r_1->is_stack()) {
 860       // Convert stack slot to an SP offset (+ wordSize to account for return address )
 861       int st_off = regs[i].first()->reg2stack() * VMRegImpl::stack_slot_size;
 862       if (!r_2->is_valid()) {
 863         // sign extend???
 864         __ ldrsw(rscratch2, Address(esp, ld_off));
 865         __ str(rscratch2, Address(sp, st_off));
 866       } else {
 867         //
 868         // We are using two optoregs. This can be either T_OBJECT,
 869         // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 870         // two slots but only uses one for thr T_LONG or T_DOUBLE case
 871         // So we must adjust where to pick up the data to match the
 872         // interpreter.
 873         //
 874         // Interpreter local[n] == MSW, local[n+1] == LSW however locals
 875         // are accessed as negative so LSW is at LOW address
 876 
 877         // ld_off is MSW so get LSW
 878         const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : ld_off;

 879         __ ldr(rscratch2, Address(esp, offset));
 880         // st_off is LSW (i.e. reg.first())
 881          __ str(rscratch2, Address(sp, st_off));
 882        }
 883      } else if (r_1->is_Register()) {  // Register argument
 884        Register r = r_1->as_Register();
 885        if (r_2->is_valid()) {
 886          //
 887          // We are using two VMRegs. This can be either T_OBJECT,
 888          // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 889          // two slots but only uses one for thr T_LONG or T_DOUBLE case
 890          // So we must adjust where to pick up the data to match the
 891          // interpreter.
 892 
 893         const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : ld_off;
 894 
 895          // this can be a misaligned move
 896          __ ldr(r, Address(esp, offset));
 897        } else {
 898          // sign extend and use a full word?
 899          __ ldrw(r, Address(esp, ld_off));
 900        }
 901      } else {
 902        if (!r_2->is_valid()) {
 903          __ ldrs(r_1->as_FloatRegister(), Address(esp, ld_off));
 904        } else {
 905          __ ldrd(r_1->as_FloatRegister(), Address(esp, next_off));
 906        }
 907      }
 908    }
 909 

















 910 
 911   // 6243940 We might end up in handle_wrong_method if
 912   // the callee is deoptimized as we race thru here. If that
 913   // happens we don't want to take a safepoint because the
 914   // caller frame will look interpreted and arguments are now
 915   // "compiled" so it is much better to make this transition
 916   // invisible to the stack walking code. Unfortunately if
 917   // we try and find the callee by normal means a safepoint
 918   // is possible. So we stash the desired callee in the thread
 919   // and the vm will find there should this case occur.
 920 
 921   __ str(rmethod, Address(rthread, JavaThread::callee_target_offset()));

 922   __ br(rscratch1);
 923 }
 924 
 925 static void gen_inline_cache_check(MacroAssembler *masm, Label& skip_fixup) {












 926 
 927   Label ok;
 928 
 929   Register holder = rscratch2;
 930   Register receiver = j_rarg0;
 931   Register tmp = r10;  // A call-clobbered register not used for arg passing
 932 
 933   // -------------------------------------------------------------------------
 934   // Generate a C2I adapter.  On entry we know rmethod holds the Method* during calls
 935   // to the interpreter.  The args start out packed in the compiled layout.  They
 936   // need to be unpacked into the interpreter layout.  This will almost always
 937   // require some stack space.  We grow the current (compiled) stack, then repack
 938   // the args.  We  finally end in a jump to the generic interpreter entry point.
 939   // On exit from the interpreter, the interpreter will restore our SP (lest the
 940   // compiled code, which relys solely on SP and not FP, get sick).
 941 
 942   {
 943     __ block_comment("c2i_unverified_entry {");
 944     __ load_klass(rscratch1, receiver);
 945     __ ldr(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
 946     __ cmp(rscratch1, tmp);
 947     __ ldr(rmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
 948     __ br(Assembler::EQ, ok);
 949     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 950 
 951     __ bind(ok);
 952     // Method might have been compiled since the call site was patched to
 953     // interpreted; if that is the case treat it as a miss so we can get
 954     // the call site corrected.
 955     __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 956     __ cbz(rscratch1, skip_fixup);
 957     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 958     __ block_comment("} c2i_unverified_entry");
 959   }
 960 }
 961 
 962 
 963 // ---------------------------------------------------------------
 964 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler* masm,
 965                                                             int comp_args_on_stack,
 966                                                             const GrowableArray<SigEntry>* sig,
 967                                                             const VMRegPair* regs,
 968                                                             const GrowableArray<SigEntry>* sig_cc,
 969                                                             const VMRegPair* regs_cc,
 970                                                             const GrowableArray<SigEntry>* sig_cc_ro,
 971                                                             const VMRegPair* regs_cc_ro,
 972                                                             AdapterFingerPrint* fingerprint,
 973                                                             AdapterBlob*& new_adapter,
 974                                                             bool allocate_code_blob) {
 975 
 976   address i2c_entry = __ pc();
 977   gen_i2c_adapter(masm, comp_args_on_stack, sig, regs);
 978 
 979   address c2i_unverified_entry = __ pc();
 980   Label skip_fixup;
 981 
 982   gen_inline_cache_check(masm, skip_fixup);
 983 
 984   OopMapSet* oop_maps = new OopMapSet();
 985   int frame_complete = CodeOffsets::frame_never_safe;
 986   int frame_size_in_words = 0;
 987 
 988   // Scalarized c2i adapter with non-scalarized receiver (i.e., don't pack receiver)
 989   address c2i_inline_ro_entry = __ pc();
 990   if (regs_cc != regs_cc_ro) {
 991     gen_c2i_adapter(masm, sig_cc_ro, regs_cc_ro, skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, false);
 992     skip_fixup.reset();
 993   }
 994 
 995   // Scalarized c2i adapter
 996   address c2i_entry = __ pc();
 997 
 998   // Class initialization barrier for static methods
 999   address c2i_no_clinit_check_entry = NULL;
1000   if (VM_Version::supports_fast_class_init_checks()) {
1001     Label L_skip_barrier;
1002 
1003     { // Bypass the barrier for non-static methods
1004       __ ldrw(rscratch1, Address(rmethod, Method::access_flags_offset()));
1005       __ andsw(zr, rscratch1, JVM_ACC_STATIC);
1006       __ br(Assembler::EQ, L_skip_barrier); // non-static
1007     }
1008 
1009     __ load_method_holder(rscratch2, rmethod);
1010     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1011     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1012 
1013     __ bind(L_skip_barrier);
1014     c2i_no_clinit_check_entry = __ pc();
1015   }
1016 
1017   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1018   bs->c2i_entry_barrier(masm);
1019 
1020   gen_c2i_adapter(masm, sig_cc, regs_cc, skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, true);
1021 
1022   address c2i_unverified_inline_entry = c2i_unverified_entry;
1023 
1024   // Non-scalarized c2i adapter
1025   address c2i_inline_entry = c2i_entry;
1026   if (regs != regs_cc) {
1027     Label inline_entry_skip_fixup;
1028     c2i_unverified_inline_entry = __ pc();
1029     gen_inline_cache_check(masm, inline_entry_skip_fixup);
1030 
1031     c2i_inline_entry = __ pc();
1032     gen_c2i_adapter(masm, sig, regs, inline_entry_skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, false);
1033   }
1034 
1035   __ flush();
1036 
1037   // The c2i adapter might safepoint and trigger a GC. The caller must make sure that
1038   // the GC knows about the location of oop argument locations passed to the c2i adapter.
1039   if (allocate_code_blob) {
1040     bool caller_must_gc_arguments = (regs != regs_cc);
1041     new_adapter = AdapterBlob::create(masm->code(), frame_complete, frame_size_in_words, oop_maps, caller_must_gc_arguments);
1042   }
1043 
1044   return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_inline_entry, c2i_inline_ro_entry, c2i_unverified_entry, c2i_unverified_inline_entry, c2i_no_clinit_check_entry);
1045 }
1046 
1047 static int c_calling_convention_priv(const BasicType *sig_bt,
1048                                          VMRegPair *regs,
1049                                          VMRegPair *regs2,
1050                                          int total_args_passed) {
1051   assert(regs2 == NULL, "not needed on AArch64");
1052 
1053 // We return the amount of VMRegImpl stack slots we need to reserve for all
1054 // the arguments NOT counting out_preserve_stack_slots.
1055 
1056     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
1057       c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5,  c_rarg6,  c_rarg7
1058     };
1059     static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
1060       c_farg0, c_farg1, c_farg2, c_farg3,
1061       c_farg4, c_farg5, c_farg6, c_farg7
1062     };
1063 
1064     uint int_args = 0;

1072       case T_BYTE:
1073       case T_SHORT:
1074       case T_INT:
1075         if (int_args < Argument::n_int_register_parameters_c) {
1076           regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
1077         } else {
1078 #ifdef __APPLE__
1079           // Less-than word types are stored one after another.
1080           // The code is unable to handle this so bailout.
1081           return -1;
1082 #endif
1083           regs[i].set1(VMRegImpl::stack2reg(stk_args));
1084           stk_args += 2;
1085         }
1086         break;
1087       case T_LONG:
1088         assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
1089         // fall through
1090       case T_OBJECT:
1091       case T_ARRAY:
1092       case T_INLINE_TYPE:
1093       case T_ADDRESS:
1094       case T_METADATA:
1095         if (int_args < Argument::n_int_register_parameters_c) {
1096           regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
1097         } else {
1098           regs[i].set2(VMRegImpl::stack2reg(stk_args));
1099           stk_args += 2;
1100         }
1101         break;
1102       case T_FLOAT:
1103         if (fp_args < Argument::n_float_register_parameters_c) {
1104           regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
1105         } else {
1106 #ifdef __APPLE__
1107           // Less-than word types are stored one after another.
1108           // The code is unable to handle this so bailout.
1109           return -1;
1110 #endif
1111           regs[i].set1(VMRegImpl::stack2reg(stk_args));
1112           stk_args += 2;

1913       reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1914     } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1915       freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1916     }
1917 #endif /* ASSERT */
1918     switch (in_sig_bt[i]) {
1919       case T_ARRAY:
1920         if (is_critical_native) {
1921           unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]);
1922           c_arg++;
1923 #ifdef ASSERT
1924           if (out_regs[c_arg].first()->is_Register()) {
1925             reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1926           } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1927             freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1928           }
1929 #endif
1930           int_args++;
1931           break;
1932         }
1933       case T_INLINE_TYPE:
1934       case T_OBJECT:
1935         assert(!is_critical_native, "no oop arguments");
1936         object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
1937                     ((i == 0) && (!is_static)),
1938                     &receiver_offset);
1939         int_args++;
1940         break;
1941       case T_VOID:
1942         break;
1943 
1944       case T_FLOAT:
1945         float_move(masm, in_regs[i], out_regs[c_arg]);
1946         float_args++;
1947         break;
1948 
1949       case T_DOUBLE:
1950         assert( i + 1 < total_in_args &&
1951                 in_sig_bt[i + 1] == T_VOID &&
1952                 out_sig_bt[c_arg+1] == T_VOID, "bad arg list");
1953         double_move(masm, in_regs[i], out_regs[c_arg]);

2030   Label lock_done;
2031 
2032   if (method->is_synchronized()) {
2033     assert(!is_critical_native, "unhandled");
2034 
2035     const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
2036 
2037     // Get the handle (the 2nd argument)
2038     __ mov(oop_handle_reg, c_rarg1);
2039 
2040     // Get address of the box
2041 
2042     __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
2043 
2044     // Load the oop from the handle
2045     __ ldr(obj_reg, Address(oop_handle_reg, 0));
2046 
2047     // Load (object->mark() | 1) into swap_reg %r0
2048     __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2049     __ orr(swap_reg, rscratch1, 1);
2050     if (EnableValhalla) {
2051       // Mask inline_type bit such that we go to the slow path if object is an inline type
2052       __ andr(swap_reg, swap_reg, ~((int) markWord::inline_type_bit_in_place));
2053     }
2054 
2055     // Save (object->mark() | 1) into BasicLock's displaced header
2056     __ str(swap_reg, Address(lock_reg, mark_word_offset));
2057 
2058     // src -> dest iff dest == r0 else r0 <- dest
2059     { Label here;
2060       __ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, lock_done, /*fallthrough*/NULL);
2061     }
2062 
2063     // Hmm should this move to the slow path code area???
2064 
2065     // Test if the oopMark is an obvious stack pointer, i.e.,
2066     //  1) (mark & 3) == 0, and
2067     //  2) sp <= mark < mark + os::pagesize()
2068     // These 3 tests can be done by evaluating the following
2069     // expression: ((mark - sp) & (3 - os::vm_page_size())),
2070     // assuming both stack pointer and pagesize have their
2071     // least significant 2 bits clear.
2072     // NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg
2073 

2099 
2100   rt_call(masm, native_func);
2101 
2102   __ bind(native_return);
2103 
2104   intptr_t return_pc = (intptr_t) __ pc();
2105   oop_maps->add_gc_map(return_pc - start, map);
2106 
2107   // Unpack native results.
2108   switch (ret_type) {
2109   case T_BOOLEAN: __ c2bool(r0);                     break;
2110   case T_CHAR   : __ ubfx(r0, r0, 0, 16);            break;
2111   case T_BYTE   : __ sbfx(r0, r0, 0, 8);             break;
2112   case T_SHORT  : __ sbfx(r0, r0, 0, 16);            break;
2113   case T_INT    : __ sbfx(r0, r0, 0, 32);            break;
2114   case T_DOUBLE :
2115   case T_FLOAT  :
2116     // Result is in v0 we'll save as needed
2117     break;
2118   case T_ARRAY:                 // Really a handle
2119   case T_INLINE_TYPE:           // Really a handle
2120   case T_OBJECT:                // Really a handle
2121       break; // can't de-handlize until after safepoint check
2122   case T_VOID: break;
2123   case T_LONG: break;
2124   default       : ShouldNotReachHere();
2125   }
2126 
2127   Label safepoint_in_progress, safepoint_in_progress_done;
2128   Label after_transition;
2129 
2130   // If this is a critical native, check for a safepoint or suspend request after the call.
2131   // If a safepoint is needed, transition to native, then to native_trans to handle
2132   // safepoints like the native methods that are not critical natives.
2133   if (is_critical_native) {
2134     Label needs_safepoint;
2135     __ safepoint_poll(needs_safepoint, false /* at_return */, true /* acquire */, false /* in_nmethod */);
2136     __ ldrw(rscratch1, Address(rthread, JavaThread::suspend_flags_offset()));
2137     __ cbnzw(rscratch1, needs_safepoint);
2138     __ b(after_transition);
2139     __ bind(needs_safepoint);

3342 #ifdef ASSERT
3343   __ str(zr, Address(rthread, JavaThread::exception_handler_pc_offset()));
3344   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
3345 #endif
3346   // Clear the exception oop so GC no longer processes it as a root.
3347   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
3348 
3349   // r0: exception oop
3350   // r8:  exception handler
3351   // r4: exception pc
3352   // Jump to handler
3353 
3354   __ br(r8);
3355 
3356   // Make sure all code is generated
3357   masm->flush();
3358 
3359   // Set exception blob
3360   _exception_blob =  ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
3361 }
3362 #endif // COMPILER2
3363 
3364 BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(const InlineKlass* vk) {
3365   BufferBlob* buf = BufferBlob::create("inline types pack/unpack", 16 * K);
3366   CodeBuffer buffer(buf);
3367   short buffer_locs[20];
3368   buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
3369                                          sizeof(buffer_locs)/sizeof(relocInfo));
3370 
3371   MacroAssembler _masm(&buffer);
3372   MacroAssembler* masm = &_masm;
3373 
3374   const Array<SigEntry>* sig_vk = vk->extended_sig();
3375   const Array<VMRegPair>* regs = vk->return_regs();
3376 
3377   int pack_fields_jobject_off = __ offset();
3378   // Resolve pre-allocated buffer from JNI handle.
3379   // We cannot do this in generate_call_stub() because it requires GC code to be initialized.
3380   Register Rresult = r14;  // See StubGenerator::generate_call_stub().
3381   __ ldr(r0, Address(Rresult));
3382   __ resolve_jobject(r0 /* value */,
3383                      rthread /* thread */,
3384                      r12 /* tmp */);
3385   __ str(r0, Address(Rresult));
3386 
3387   int pack_fields_off = __ offset();
3388 
3389   int j = 1;
3390   for (int i = 0; i < sig_vk->length(); i++) {
3391     BasicType bt = sig_vk->at(i)._bt;
3392     if (bt == T_INLINE_TYPE) {
3393       continue;
3394     }
3395     if (bt == T_VOID) {
3396       if (sig_vk->at(i-1)._bt == T_LONG ||
3397           sig_vk->at(i-1)._bt == T_DOUBLE) {
3398         j++;
3399       }
3400       continue;
3401     }
3402     int off = sig_vk->at(i)._offset;
3403     VMRegPair pair = regs->at(j);
3404     VMReg r_1 = pair.first();
3405     VMReg r_2 = pair.second();
3406     Address to(r0, off);
3407     if (bt == T_FLOAT) {
3408       __ strs(r_1->as_FloatRegister(), to);
3409     } else if (bt == T_DOUBLE) {
3410       __ strd(r_1->as_FloatRegister(), to);
3411     } else if (bt == T_OBJECT || bt == T_ARRAY) {
3412       Register val = r_1->as_Register();
3413       assert_different_registers(r0, val);
3414       // We don't need barriers because the destination is a newly allocated object.
3415       // Also, we cannot use store_heap_oop(to, val) because it uses r8 as tmp.
3416       if (UseCompressedOops) {
3417         __ encode_heap_oop(val);
3418         __ str(val, to);
3419       } else {
3420         __ str(val, to);
3421       }
3422     } else {
3423       assert(is_java_primitive(bt), "unexpected basic type");
3424       assert_different_registers(r0, r_1->as_Register());
3425       size_t size_in_bytes = type2aelembytes(bt);
3426       __ store_sized_value(to, r_1->as_Register(), size_in_bytes);
3427     }
3428     j++;
3429   }
3430   assert(j == regs->length(), "missed a field?");
3431 
3432   __ ret(lr);
3433 
3434   int unpack_fields_off = __ offset();
3435 
3436   j = 1;
3437   for (int i = 0; i < sig_vk->length(); i++) {
3438     BasicType bt = sig_vk->at(i)._bt;
3439     if (bt == T_INLINE_TYPE) {
3440       continue;
3441     }
3442     if (bt == T_VOID) {
3443       if (sig_vk->at(i-1)._bt == T_LONG ||
3444           sig_vk->at(i-1)._bt == T_DOUBLE) {
3445         j++;
3446       }
3447       continue;
3448     }
3449     int off = sig_vk->at(i)._offset;
3450     assert(off > 0, "offset in object should be positive");
3451     VMRegPair pair = regs->at(j);
3452     VMReg r_1 = pair.first();
3453     VMReg r_2 = pair.second();
3454     Address from(r0, off);
3455     if (bt == T_FLOAT) {
3456       __ ldrs(r_1->as_FloatRegister(), from);
3457     } else if (bt == T_DOUBLE) {
3458       __ ldrd(r_1->as_FloatRegister(), from);
3459     } else if (bt == T_OBJECT || bt == T_ARRAY) {
3460       assert_different_registers(r0, r_1->as_Register());
3461       __ load_heap_oop(r_1->as_Register(), from);
3462     } else {
3463       assert(is_java_primitive(bt), "unexpected basic type");
3464       assert_different_registers(r0, r_1->as_Register());
3465 
3466       size_t size_in_bytes = type2aelembytes(bt);
3467       __ load_sized_value(r_1->as_Register(), from, size_in_bytes, bt != T_CHAR && bt != T_BOOLEAN);
3468     }
3469     j++;
3470   }
3471   assert(j == regs->length(), "missed a field?");
3472 
3473   if (StressInlineTypeReturnedAsFields) {
3474     __ load_klass(r0, r0);
3475     __ orr(r0, r0, 1);
3476   }
3477 
3478   __ ret(lr);
3479 
3480   __ flush();
3481 
3482   return BufferedInlineTypeBlob::create(&buffer, pack_fields_off, pack_fields_jobject_off, unpack_fields_off);
3483 }
3484 
3485 // ---------------------------------------------------------------
3486 
3487 class NativeInvokerGenerator : public StubCodeGenerator {
3488   address _call_target;
3489   int _shadow_space_bytes;
3490 
3491   const GrowableArray<VMReg>& _input_registers;
3492   const GrowableArray<VMReg>& _output_registers;
3493 
3494   int _frame_complete;
3495   int _framesize;
3496   OopMapSet* _oop_maps;
3497 public:
3498   NativeInvokerGenerator(CodeBuffer* buffer,
3499                          address call_target,
3500                          int shadow_space_bytes,
3501                          const GrowableArray<VMReg>& input_registers,
3502                          const GrowableArray<VMReg>& output_registers)
3503    : StubCodeGenerator(buffer, PrintMethodHandleStubs),

3716 
3717   //////////////////////////////////////////////////////////////////////////////
3718 
3719   __ block_comment("{ L_reguard");
3720   __ bind(L_reguard);
3721 
3722   spill_output_registers();
3723 
3724   rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
3725 
3726   fill_output_registers();
3727 
3728   __ b(L_after_reguard);
3729 
3730   __ block_comment("} L_reguard");
3731 
3732   //////////////////////////////////////////////////////////////////////////////
3733 
3734   __ flush();
3735 }

< prev index next >