< prev index next >

src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp

Print this page

   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "asm/macroAssembler.hpp"
  28 #include "asm/macroAssembler.inline.hpp"

  29 #include "code/aotCodeCache.hpp"
  30 #include "code/codeCache.hpp"
  31 #include "code/compiledIC.hpp"
  32 #include "code/debugInfoRec.hpp"
  33 #include "code/vtableStubs.hpp"
  34 #include "compiler/oopMap.hpp"
  35 #include "gc/shared/barrierSetAssembler.hpp"
  36 #include "interpreter/interpreter.hpp"
  37 #include "interpreter/interp_masm.hpp"
  38 #include "logging/log.hpp"
  39 #include "memory/resourceArea.hpp"
  40 #include "nativeInst_aarch64.hpp"
  41 #include "oops/klass.inline.hpp"
  42 #include "oops/method.inline.hpp"
  43 #include "prims/methodHandles.hpp"
  44 #include "runtime/continuation.hpp"
  45 #include "runtime/continuationEntry.inline.hpp"
  46 #include "runtime/globals.hpp"
  47 #include "runtime/jniHandles.hpp"
  48 #include "runtime/safepointMechanism.hpp"

 342       break;
 343     case T_DOUBLE:
 344       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 345       if (fp_args < Argument::n_float_register_parameters_j) {
 346         regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
 347       } else {
 348         stk_args = align_up(stk_args, 2);
 349         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 350         stk_args += 2;
 351       }
 352       break;
 353     default:
 354       ShouldNotReachHere();
 355       break;
 356     }
 357   }
 358 
 359   return stk_args;
 360 }
 361 















































































 362 // Patch the callers callsite with entry to compiled code if it exists.
 363 static void patch_callers_callsite(MacroAssembler *masm) {
 364   Label L;
 365   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 366   __ cbz(rscratch1, L);
 367 
 368   __ enter();
 369   __ push_CPU_state();
 370 
 371   // VM needs caller's callsite
 372   // VM needs target method
 373   // This needs to be a long call since we will relocate this adapter to
 374   // the codeBuffer and it may not reach
 375 
 376 #ifndef PRODUCT
 377   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
 378 #endif
 379 
 380   __ mov(c_rarg0, rmethod);
 381   __ mov(c_rarg1, lr);
 382   __ authenticate_return_address(c_rarg1);
 383   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
 384   __ blr(rscratch1);
 385 
 386   // Explicit isb required because fixup_callers_callsite may change the code
 387   // stream.
 388   __ safepoint_isb();
 389 
 390   __ pop_CPU_state();
 391   // restore sp
 392   __ leave();
 393   __ bind(L);
 394 }
 395 











































































































 396 static void gen_c2i_adapter(MacroAssembler *masm,
 397                             int total_args_passed,
 398                             int comp_args_on_stack,
 399                             const BasicType *sig_bt,
 400                             const VMRegPair *regs,
 401                             Label& skip_fixup) {



























 402   // Before we get into the guts of the C2I adapter, see if we should be here
 403   // at all.  We've come from compiled code and are attempting to jump to the
 404   // interpreter, which means the caller made a static call to get here
 405   // (vcalls always get a compiled target if there is one).  Check for a
 406   // compiled target.  If there is one, we need to patch the caller's call.
 407   patch_callers_callsite(masm);
 408 
 409   __ bind(skip_fixup);
 410 
 411   int words_pushed = 0;
























 412 
 413   // Since all args are passed on the stack, total_args_passed *
 414   // Interpreter::stackElementSize is the space we need.
 415 
 416   int extraspace = total_args_passed * Interpreter::stackElementSize;

 417 
 418   __ mov(r19_sender_sp, sp);


 419 
 420   // stack is aligned, keep it that way
 421   extraspace = align_up(extraspace, 2*wordSize);

 422 
 423   if (extraspace)
 424     __ sub(sp, sp, extraspace);
 425 
 426   // Now write the args into the outgoing interpreter space
 427   for (int i = 0; i < total_args_passed; i++) {
 428     if (sig_bt[i] == T_VOID) {
 429       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
 430       continue;
 431     }
 432 
 433     // offset to start parameters
 434     int st_off   = (total_args_passed - i - 1) * Interpreter::stackElementSize;
 435     int next_off = st_off - Interpreter::stackElementSize;
 436 
 437     // Say 4 args:
 438     // i   st_off
 439     // 0   32 T_LONG
 440     // 1   24 T_VOID
 441     // 2   16 T_OBJECT
 442     // 3    8 T_BOOL
 443     // -    0 return address
 444     //
 445     // However to make thing extra confusing. Because we can fit a Java long/double in
 446     // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
 447     // leaves one slot empty and only stores to a single slot. In this case the
 448     // slot that is occupied is the T_VOID slot. See I said it was confusing.
 449 
 450     VMReg r_1 = regs[i].first();
 451     VMReg r_2 = regs[i].second();
 452     if (!r_1->is_valid()) {
 453       assert(!r_2->is_valid(), "");
 454       continue;




 455     }
 456     if (r_1->is_stack()) {
 457       // memory to memory use rscratch1
 458       int ld_off = (r_1->reg2stack() * VMRegImpl::stack_slot_size
 459                     + extraspace
 460                     + words_pushed * wordSize);
 461       if (!r_2->is_valid()) {
 462         // sign extend??
 463         __ ldrw(rscratch1, Address(sp, ld_off));
 464         __ str(rscratch1, Address(sp, st_off));
 465 
 466       } else {

 467 
 468         __ ldr(rscratch1, Address(sp, ld_off));




 469 
 470         // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
 471         // T_DOUBLE and T_LONG use two slots in the interpreter
 472         if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
 473           // ld_off == LSW, ld_off+wordSize == MSW
 474           // st_off == MSW, next_off == LSW
 475           __ str(rscratch1, Address(sp, next_off));

























 476 #ifdef ASSERT
 477           // Overwrite the unused slot with known junk
 478           __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaaaull);
 479           __ str(rscratch1, Address(sp, st_off));
 480 #endif /* ASSERT */
 481         } else {
 482           __ str(rscratch1, Address(sp, st_off));
 483         }
 484       }
 485     } else if (r_1->is_Register()) {
 486       Register r = r_1->as_Register();
 487       if (!r_2->is_valid()) {
 488         // must be only an int (or less ) so move only 32bits to slot
 489         // why not sign extend??
 490         __ str(r, Address(sp, st_off));
 491       } else {
 492         // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
 493         // T_DOUBLE and T_LONG use two slots in the interpreter
 494         if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
 495           // jlong/double in gpr
 496 #ifdef ASSERT
 497           // Overwrite the unused slot with known junk
 498           __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaabull);
 499           __ str(rscratch1, Address(sp, st_off));
 500 #endif /* ASSERT */
 501           __ str(r, Address(sp, next_off));























 502         } else {
 503           __ str(r, Address(sp, st_off));





















 504         }
 505       }
 506     } else {
 507       assert(r_1->is_FloatRegister(), "");
 508       if (!r_2->is_valid()) {
 509         // only a float use just part of the slot
 510         __ strs(r_1->as_FloatRegister(), Address(sp, st_off));
 511       } else {
 512 #ifdef ASSERT
 513         // Overwrite the unused slot with known junk
 514         __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaacull);
 515         __ str(rscratch1, Address(sp, st_off));
 516 #endif /* ASSERT */
 517         __ strd(r_1->as_FloatRegister(), Address(sp, next_off));
 518       }
 519     }
 520   }
 521 
 522   __ mov(esp, sp); // Interp expects args on caller's expression stack
 523 
 524   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));
 525   __ br(rscratch1);
 526 }
 527 

 528 
 529 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
 530                                     int total_args_passed,
 531                                     int comp_args_on_stack,
 532                                     const BasicType *sig_bt,
 533                                     const VMRegPair *regs) {
 534 
 535   // Note: r19_sender_sp contains the senderSP on entry. We must
 536   // preserve it since we may do a i2c -> c2i transition if we lose a
 537   // race where compiled code goes non-entrant while we get args
 538   // ready.
 539 
 540   // Adapters are frameless.
 541 
 542   // An i2c adapter is frameless because the *caller* frame, which is
 543   // interpreted, routinely repairs its own esp (from
 544   // interpreter_frame_last_sp), even if a callee has modified the
 545   // stack pointer.  It also recalculates and aligns sp.
 546 
 547   // A c2i adapter is frameless because the *callee* frame, which is
 548   // interpreted, routinely repairs its caller's sp (from sender_sp,
 549   // which is set up via the senderSP register).
 550 
 551   // In other words, if *either* the caller or callee is interpreted, we can
 552   // get the stack pointer repaired after a call.
 553 
 554   // This is why c2i and i2c adapters cannot be indefinitely composed.
 555   // In particular, if a c2i adapter were to somehow call an i2c adapter,
 556   // both caller and callee would be compiled methods, and neither would
 557   // clean up the stack pointer changes performed by the two adapters.
 558   // If this happens, control eventually transfers back to the compiled
 559   // caller, but with an uncorrected stack, causing delayed havoc.
 560 
 561   // Cut-out for having no stack args.
 562   int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord;
 563   if (comp_args_on_stack) {
 564     __ sub(rscratch1, sp, comp_words_on_stack * wordSize);
 565     __ andr(sp, rscratch1, -16);

 566   }
 567 
 568   // Will jump to the compiled code just as if compiled code was doing it.
 569   // Pre-load the register-jump target early, to schedule it better.
 570   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::from_compiled_offset())));
 571 
 572 #if INCLUDE_JVMCI
 573   if (EnableJVMCI) {
 574     // check if this call should be routed towards a specific entry point
 575     __ ldr(rscratch2, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 576     Label no_alternative_target;
 577     __ cbz(rscratch2, no_alternative_target);
 578     __ mov(rscratch1, rscratch2);
 579     __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 580     __ bind(no_alternative_target);
 581   }
 582 #endif // INCLUDE_JVMCI
 583 


 584   // Now generate the shuffle code.
 585   for (int i = 0; i < total_args_passed; i++) {
 586     if (sig_bt[i] == T_VOID) {
 587       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");

 588       continue;
 589     }
 590 
 591     // Pick up 0, 1 or 2 words from SP+offset.

 592 
 593     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(),
 594             "scrambled load targets?");
 595     // Load in argument order going down.
 596     int ld_off = (total_args_passed - i - 1)*Interpreter::stackElementSize;
 597     // Point to interpreter value (vs. tag)
 598     int next_off = ld_off - Interpreter::stackElementSize;
 599     //
 600     //
 601     //
 602     VMReg r_1 = regs[i].first();
 603     VMReg r_2 = regs[i].second();
 604     if (!r_1->is_valid()) {
 605       assert(!r_2->is_valid(), "");
 606       continue;
 607     }
 608     if (r_1->is_stack()) {
 609       // Convert stack slot to an SP offset (+ wordSize to account for return address )
 610       int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size;
 611       if (!r_2->is_valid()) {
 612         // sign extend???
 613         __ ldrsw(rscratch2, Address(esp, ld_off));
 614         __ str(rscratch2, Address(sp, st_off));
 615       } else {
 616         //
 617         // We are using two optoregs. This can be either T_OBJECT,
 618         // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 619         // two slots but only uses one for thr T_LONG or T_DOUBLE case
 620         // So we must adjust where to pick up the data to match the
 621         // interpreter.
 622         //
 623         // Interpreter local[n] == MSW, local[n+1] == LSW however locals
 624         // are accessed as negative so LSW is at LOW address
 625 
 626         // ld_off is MSW so get LSW
 627         const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
 628                            next_off : ld_off;
 629         __ ldr(rscratch2, Address(esp, offset));
 630         // st_off is LSW (i.e. reg.first())
 631         __ str(rscratch2, Address(sp, st_off));
 632       }
 633     } else if (r_1->is_Register()) {  // Register argument
 634       Register r = r_1->as_Register();
 635       if (r_2->is_valid()) {
 636         //
 637         // We are using two VMRegs. This can be either T_OBJECT,
 638         // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 639         // two slots but only uses one for thr T_LONG or T_DOUBLE case
 640         // So we must adjust where to pick up the data to match the
 641         // interpreter.

















 642 
 643         const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
 644                            next_off : ld_off;
 645 
 646         // this can be a misaligned move
 647         __ ldr(r, Address(esp, offset));
 648       } else {
 649         // sign extend and use a full word?
 650         __ ldrw(r, Address(esp, ld_off));
 651       }
 652     } else {
 653       if (!r_2->is_valid()) {
 654         __ ldrs(r_1->as_FloatRegister(), Address(esp, ld_off));
 655       } else {
 656         __ ldrd(r_1->as_FloatRegister(), Address(esp, next_off));
 657       }
 658     }
 659   }
 660 
 661   __ mov(rscratch2, rscratch1);
 662   __ push_cont_fastpath(rthread); // Set JavaThread::_cont_fastpath to the sp of the oldest interpreted frame we know about; kills rscratch1
 663   __ mov(rscratch1, rscratch2);
 664 
 665   // 6243940 We might end up in handle_wrong_method if
 666   // the callee is deoptimized as we race thru here. If that
 667   // happens we don't want to take a safepoint because the
 668   // caller frame will look interpreted and arguments are now
 669   // "compiled" so it is much better to make this transition
 670   // invisible to the stack walking code. Unfortunately if
 671   // we try and find the callee by normal means a safepoint
 672   // is possible. So we stash the desired callee in the thread
 673   // and the vm will find there should this case occur.
 674 
 675   __ str(rmethod, Address(rthread, JavaThread::callee_target_offset()));
 676 
 677   __ br(rscratch1);
 678 }
 679 













 680 // ---------------------------------------------------------------
 681 void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
 682                                             int total_args_passed,
 683                                             int comp_args_on_stack,
 684                                             const BasicType *sig_bt,
 685                                             const VMRegPair *regs,
 686                                             AdapterHandlerEntry* handler) {






 687   address i2c_entry = __ pc();
 688 
 689   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
 690 
 691   address c2i_unverified_entry = __ pc();
 692   Label skip_fixup;
 693 
 694   Register data = rscratch2;
 695   Register receiver = j_rarg0;
 696   Register tmp = r10;  // A call-clobbered register not used for arg passing
 697 
 698   // -------------------------------------------------------------------------
 699   // Generate a C2I adapter.  On entry we know rmethod holds the Method* during calls
 700   // to the interpreter.  The args start out packed in the compiled layout.  They
 701   // need to be unpacked into the interpreter layout.  This will almost always
 702   // require some stack space.  We grow the current (compiled) stack, then repack
 703   // the args.  We  finally end in a jump to the generic interpreter entry point.
 704   // On exit from the interpreter, the interpreter will restore our SP (lest the
 705   // compiled code, which relies solely on SP and not FP, get sick).
 706 
 707   {
 708     __ block_comment("c2i_unverified_entry {");
 709     // Method might have been compiled since the call site was patched to
 710     // interpreted; if that is the case treat it as a miss so we can get
 711     // the call site corrected.
 712     __ ic_check(1 /* end_alignment */);
 713     __ ldr(rmethod, Address(data, CompiledICData::speculated_method_offset()));
 714 
 715     __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 716     __ cbz(rscratch1, skip_fixup);
 717     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 718     __ block_comment("} c2i_unverified_entry");
 719   }
 720 
 721   address c2i_entry = __ pc();


 722 
 723   // Class initialization barrier for static methods
 724   address c2i_no_clinit_check_entry = nullptr;
 725   if (VM_Version::supports_fast_class_init_checks()) {
 726     Label L_skip_barrier;





 727 
 728     { // Bypass the barrier for non-static methods
 729       __ ldrh(rscratch1, Address(rmethod, Method::access_flags_offset()));
 730       __ andsw(zr, rscratch1, JVM_ACC_STATIC);
 731       __ br(Assembler::EQ, L_skip_barrier); // non-static
 732     }
 733 
 734     __ load_method_holder(rscratch2, rmethod);
 735     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 736     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));


 737 
 738     __ bind(L_skip_barrier);
 739     c2i_no_clinit_check_entry = __ pc();

 740   }
 741 
 742   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 743   bs->c2i_entry_barrier(masm);
 744 
 745   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);


 746 
 747   handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
 748   return;
 749 }
 750 
 751 static int c_calling_convention_priv(const BasicType *sig_bt,
 752                                          VMRegPair *regs,
 753                                          int total_args_passed) {
 754 
 755 // We return the amount of VMRegImpl stack slots we need to reserve for all
 756 // the arguments NOT counting out_preserve_stack_slots.
 757 
 758     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
 759       c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5,  c_rarg6,  c_rarg7
 760     };
 761     static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
 762       c_farg0, c_farg1, c_farg2, c_farg3,
 763       c_farg4, c_farg5, c_farg6, c_farg7
 764     };
 765 
 766     uint int_args = 0;
 767     uint fp_args = 0;
 768     uint stk_args = 0; // inc by 2 each time

1765   if (method->is_synchronized()) {
1766     Label count;
1767     const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
1768 
1769     // Get the handle (the 2nd argument)
1770     __ mov(oop_handle_reg, c_rarg1);
1771 
1772     // Get address of the box
1773 
1774     __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
1775 
1776     // Load the oop from the handle
1777     __ ldr(obj_reg, Address(oop_handle_reg, 0));
1778 
1779     if (LockingMode == LM_MONITOR) {
1780       __ b(slow_path_lock);
1781     } else if (LockingMode == LM_LEGACY) {
1782       // Load (object->mark() | 1) into swap_reg %r0
1783       __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1784       __ orr(swap_reg, rscratch1, 1);




1785 
1786       // Save (object->mark() | 1) into BasicLock's displaced header
1787       __ str(swap_reg, Address(lock_reg, mark_word_offset));
1788 
1789       // src -> dest iff dest == r0 else r0 <- dest
1790       __ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, count, /*fallthrough*/nullptr);
1791 
1792       // Hmm should this move to the slow path code area???
1793 
1794       // Test if the oopMark is an obvious stack pointer, i.e.,
1795       //  1) (mark & 3) == 0, and
1796       //  2) sp <= mark < mark + os::pagesize()
1797       // These 3 tests can be done by evaluating the following
1798       // expression: ((mark - sp) & (3 - os::vm_page_size())),
1799       // assuming both stack pointer and pagesize have their
1800       // least significant 2 bits clear.
1801       // NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg
1802 
1803       __ sub(swap_reg, sp, swap_reg);
1804       __ neg(swap_reg, swap_reg);

2806 
2807   // exception pending => remove activation and forward to exception handler
2808 
2809   __ str(zr, Address(rthread, JavaThread::vm_result_oop_offset()));
2810 
2811   __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
2812   __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2813 
2814   // -------------
2815   // make sure all code is generated
2816   masm->flush();
2817 
2818   // return the  blob
2819   // frame_size_words or bytes??
2820   RuntimeStub* rs_blob = RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, true);
2821 
2822   AOTCodeCache::store_code_blob(*rs_blob, AOTCodeEntry::SharedBlob, (uint)id, name);
2823   return rs_blob;
2824 }
2825 












































































































































2826 // Continuation point for throwing of implicit exceptions that are
2827 // not handled in the current activation. Fabricates an exception
2828 // oop and initiates normal exception dispatching in this
2829 // frame. Since we need to preserve callee-saved values (currently
2830 // only for C2, but done for C1 as well) we need a callee-saved oop
2831 // map and therefore have to make these stubs into RuntimeStubs
2832 // rather than BufferBlobs.  If the compiler needs all registers to
2833 // be preserved between the fault point and the exception handler
2834 // then it must assume responsibility for that in
2835 // AbstractCompiler::continuation_for_implicit_null_exception or
2836 // continuation_for_implicit_division_by_zero_exception. All other
2837 // implicit exceptions (e.g., NullPointerException or
2838 // AbstractMethodError on entry) are either at call sites or
2839 // otherwise assume that stack unwinding will be initiated, so
2840 // caller saved registers were assumed volatile in the compiler.
2841 
2842 RuntimeStub* SharedRuntime::generate_throw_exception(SharedStubId id, address runtime_entry) {
2843   assert(is_throw_id(id), "expected a throw stub id");
2844 
2845   const char* name = SharedRuntime::stub_name(id);

   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "asm/macroAssembler.hpp"
  28 #include "asm/macroAssembler.inline.hpp"
  29 #include "classfile/symbolTable.hpp"
  30 #include "code/aotCodeCache.hpp"
  31 #include "code/codeCache.hpp"
  32 #include "code/compiledIC.hpp"
  33 #include "code/debugInfoRec.hpp"
  34 #include "code/vtableStubs.hpp"
  35 #include "compiler/oopMap.hpp"
  36 #include "gc/shared/barrierSetAssembler.hpp"
  37 #include "interpreter/interpreter.hpp"
  38 #include "interpreter/interp_masm.hpp"
  39 #include "logging/log.hpp"
  40 #include "memory/resourceArea.hpp"
  41 #include "nativeInst_aarch64.hpp"
  42 #include "oops/klass.inline.hpp"
  43 #include "oops/method.inline.hpp"
  44 #include "prims/methodHandles.hpp"
  45 #include "runtime/continuation.hpp"
  46 #include "runtime/continuationEntry.inline.hpp"
  47 #include "runtime/globals.hpp"
  48 #include "runtime/jniHandles.hpp"
  49 #include "runtime/safepointMechanism.hpp"

 343       break;
 344     case T_DOUBLE:
 345       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 346       if (fp_args < Argument::n_float_register_parameters_j) {
 347         regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
 348       } else {
 349         stk_args = align_up(stk_args, 2);
 350         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 351         stk_args += 2;
 352       }
 353       break;
 354     default:
 355       ShouldNotReachHere();
 356       break;
 357     }
 358   }
 359 
 360   return stk_args;
 361 }
 362 
 363 
 364 const uint SharedRuntime::java_return_convention_max_int = Argument::n_int_register_parameters_j;
 365 const uint SharedRuntime::java_return_convention_max_float = Argument::n_float_register_parameters_j;
 366 
 367 int SharedRuntime::java_return_convention(const BasicType *sig_bt, VMRegPair *regs, int total_args_passed) {
 368 
 369   // Create the mapping between argument positions and registers.
 370 
 371   static const Register INT_ArgReg[java_return_convention_max_int] = {
 372     r0 /* j_rarg7 */, j_rarg6, j_rarg5, j_rarg4, j_rarg3, j_rarg2, j_rarg1, j_rarg0
 373   };
 374 
 375   static const FloatRegister FP_ArgReg[java_return_convention_max_float] = {
 376     j_farg0, j_farg1, j_farg2, j_farg3, j_farg4, j_farg5, j_farg6, j_farg7
 377   };
 378 
 379   uint int_args = 0;
 380   uint fp_args = 0;
 381 
 382   for (int i = 0; i < total_args_passed; i++) {
 383     switch (sig_bt[i]) {
 384     case T_BOOLEAN:
 385     case T_CHAR:
 386     case T_BYTE:
 387     case T_SHORT:
 388     case T_INT:
 389       if (int_args < SharedRuntime::java_return_convention_max_int) {
 390         regs[i].set1(INT_ArgReg[int_args]->as_VMReg());
 391         int_args ++;
 392       } else {
 393         return -1;
 394       }
 395       break;
 396     case T_VOID:
 397       // halves of T_LONG or T_DOUBLE
 398       assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
 399       regs[i].set_bad();
 400       break;
 401     case T_LONG:
 402       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 403       // fall through
 404     case T_OBJECT:
 405     case T_ARRAY:
 406     case T_ADDRESS:
 407       // Should T_METADATA be added to java_calling_convention as well ?
 408     case T_METADATA:
 409       if (int_args < SharedRuntime::java_return_convention_max_int) {
 410         regs[i].set2(INT_ArgReg[int_args]->as_VMReg());
 411         int_args ++;
 412       } else {
 413         return -1;
 414       }
 415       break;
 416     case T_FLOAT:
 417       if (fp_args < SharedRuntime::java_return_convention_max_float) {
 418         regs[i].set1(FP_ArgReg[fp_args]->as_VMReg());
 419         fp_args ++;
 420       } else {
 421         return -1;
 422       }
 423       break;
 424     case T_DOUBLE:
 425       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 426       if (fp_args < SharedRuntime::java_return_convention_max_float) {
 427         regs[i].set2(FP_ArgReg[fp_args]->as_VMReg());
 428         fp_args ++;
 429       } else {
 430         return -1;
 431       }
 432       break;
 433     default:
 434       ShouldNotReachHere();
 435       break;
 436     }
 437   }
 438 
 439   return int_args + fp_args;
 440 }
 441 
 442 // Patch the callers callsite with entry to compiled code if it exists.
 443 static void patch_callers_callsite(MacroAssembler *masm) {
 444   Label L;
 445   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 446   __ cbz(rscratch1, L);
 447 
 448   __ enter();
 449   __ push_CPU_state();
 450 
 451   // VM needs caller's callsite
 452   // VM needs target method
 453   // This needs to be a long call since we will relocate this adapter to
 454   // the codeBuffer and it may not reach
 455 
 456 #ifndef PRODUCT
 457   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
 458 #endif
 459 
 460   __ mov(c_rarg0, rmethod);
 461   __ mov(c_rarg1, lr);
 462   __ authenticate_return_address(c_rarg1);
 463   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
 464   __ blr(rscratch1);
 465 
 466   // Explicit isb required because fixup_callers_callsite may change the code
 467   // stream.
 468   __ safepoint_isb();
 469 
 470   __ pop_CPU_state();
 471   // restore sp
 472   __ leave();
 473   __ bind(L);
 474 }
 475 
 476 // For each inline type argument, sig includes the list of fields of
 477 // the inline type. This utility function computes the number of
 478 // arguments for the call if inline types are passed by reference (the
 479 // calling convention the interpreter expects).
 480 static int compute_total_args_passed_int(const GrowableArray<SigEntry>* sig_extended) {
 481   int total_args_passed = 0;
 482   if (InlineTypePassFieldsAsArgs) {
 483     for (int i = 0; i < sig_extended->length(); i++) {
 484       BasicType bt = sig_extended->at(i)._bt;
 485       if (bt == T_METADATA) {
 486         // In sig_extended, an inline type argument starts with:
 487         // T_METADATA, followed by the types of the fields of the
 488         // inline type and T_VOID to mark the end of the value
 489         // type. Inline types are flattened so, for instance, in the
 490         // case of an inline type with an int field and an inline type
 491         // field that itself has 2 fields, an int and a long:
 492         // T_METADATA T_INT T_METADATA T_INT T_LONG T_VOID (second
 493         // slot for the T_LONG) T_VOID (inner inline type) T_VOID
 494         // (outer inline type)
 495         total_args_passed++;
 496         int vt = 1;
 497         do {
 498           i++;
 499           BasicType bt = sig_extended->at(i)._bt;
 500           BasicType prev_bt = sig_extended->at(i-1)._bt;
 501           if (bt == T_METADATA) {
 502             vt++;
 503           } else if (bt == T_VOID &&
 504                      prev_bt != T_LONG &&
 505                      prev_bt != T_DOUBLE) {
 506             vt--;
 507           }
 508         } while (vt != 0);
 509       } else {
 510         total_args_passed++;
 511       }
 512     }
 513   } else {
 514     total_args_passed = sig_extended->length();
 515   }
 516   return total_args_passed;
 517 }
 518 
 519 
 520 static void gen_c2i_adapter_helper(MacroAssembler* masm,
 521                                    BasicType bt,
 522                                    BasicType prev_bt,
 523                                    size_t size_in_bytes,
 524                                    const VMRegPair& reg_pair,
 525                                    const Address& to,
 526                                    Register tmp1,
 527                                    Register tmp2,
 528                                    Register tmp3,
 529                                    int extraspace,
 530                                    bool is_oop) {
 531   if (bt == T_VOID) {
 532     assert(prev_bt == T_LONG || prev_bt == T_DOUBLE, "missing half");
 533     return;
 534   }
 535 
 536   // Say 4 args:
 537   // i   st_off
 538   // 0   32 T_LONG
 539   // 1   24 T_VOID
 540   // 2   16 T_OBJECT
 541   // 3    8 T_BOOL
 542   // -    0 return address
 543   //
 544   // However to make thing extra confusing. Because we can fit a Java long/double in
 545   // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
 546   // leaves one slot empty and only stores to a single slot. In this case the
 547   // slot that is occupied is the T_VOID slot. See I said it was confusing.
 548 
 549   bool wide = (size_in_bytes == wordSize);
 550   VMReg r_1 = reg_pair.first();
 551   VMReg r_2 = reg_pair.second();
 552   assert(r_2->is_valid() == wide, "invalid size");
 553   if (!r_1->is_valid()) {
 554     assert(!r_2->is_valid(), "");
 555     return;
 556   }
 557 
 558   if (!r_1->is_FloatRegister()) {
 559     Register val = r25;
 560     if (r_1->is_stack()) {
 561       // memory to memory use r25 (scratch registers is used by store_heap_oop)
 562       int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
 563       __ load_sized_value(val, Address(sp, ld_off), size_in_bytes, /* is_signed */ false);
 564     } else {
 565       val = r_1->as_Register();
 566     }
 567     assert_different_registers(to.base(), val, tmp1, tmp2, tmp3);
 568     if (is_oop) {
 569       __ store_heap_oop(to, val, tmp1, tmp2, tmp3, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
 570     } else {
 571       __ store_sized_value(to, val, size_in_bytes);
 572     }
 573   } else {
 574     if (wide) {
 575       __ strd(r_1->as_FloatRegister(), to);
 576     } else {
 577       // only a float use just part of the slot
 578       __ strs(r_1->as_FloatRegister(), to);
 579     }
 580   }
 581 }
 582 
 583 static void gen_c2i_adapter(MacroAssembler *masm,
 584                             const GrowableArray<SigEntry>* sig_extended,


 585                             const VMRegPair *regs,
 586                             bool requires_clinit_barrier,
 587                             address& c2i_no_clinit_check_entry,
 588                             Label& skip_fixup,
 589                             address start,
 590                             OopMapSet* oop_maps,
 591                             int& frame_complete,
 592                             int& frame_size_in_words,
 593                             bool alloc_inline_receiver) {
 594   if (requires_clinit_barrier && VM_Version::supports_fast_class_init_checks()) {
 595     Label L_skip_barrier;
 596 
 597     { // Bypass the barrier for non-static methods
 598       __ ldrh(rscratch1, Address(rmethod, Method::access_flags_offset()));
 599       __ andsw(zr, rscratch1, JVM_ACC_STATIC);
 600       __ br(Assembler::EQ, L_skip_barrier); // non-static
 601     }
 602 
 603     __ load_method_holder(rscratch2, rmethod);
 604     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 605     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 606 
 607     __ bind(L_skip_barrier);
 608     c2i_no_clinit_check_entry = __ pc();
 609   }
 610 
 611   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 612   bs->c2i_entry_barrier(masm);
 613 
 614   // Before we get into the guts of the C2I adapter, see if we should be here
 615   // at all.  We've come from compiled code and are attempting to jump to the
 616   // interpreter, which means the caller made a static call to get here
 617   // (vcalls always get a compiled target if there is one).  Check for a
 618   // compiled target.  If there is one, we need to patch the caller's call.
 619   patch_callers_callsite(masm);
 620 
 621   __ bind(skip_fixup);
 622 
 623   // TODO 8366717 Is the comment about r13 correct? Isn't that r19_sender_sp?
 624   // Name some registers to be used in the following code. We can use
 625   // anything except r0-r7 which are arguments in the Java calling
 626   // convention, rmethod (r12), and r13 which holds the outgoing sender
 627   // SP for the interpreter.
 628   // TODO 8366717 We need to make sure that buf_array, buf_oop (and potentially other long-life regs) are kept live in slowpath runtime calls in GC barriers
 629   Register buf_array = r10;   // Array of buffered inline types
 630   Register buf_oop = r11;     // Buffered inline type oop
 631   Register tmp1 = r15;
 632   Register tmp2 = r16;
 633   Register tmp3 = r17;
 634 
 635   if (InlineTypePassFieldsAsArgs) {
 636     // Is there an inline type argument?
 637     bool has_inline_argument = false;
 638     for (int i = 0; i < sig_extended->length() && !has_inline_argument; i++) {
 639       has_inline_argument = (sig_extended->at(i)._bt == T_METADATA);
 640     }
 641     if (has_inline_argument) {
 642       // There is at least an inline type argument: we're coming from
 643       // compiled code so we have no buffers to back the inline types
 644       // Allocate the buffers here with a runtime call.
 645       // TODO 8366717 Do we need to save vectors here? They could be used as arg registers, right? Same on x64.
 646       RegisterSaver reg_save(true /* save_vectors */);
 647       OopMap* map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
 648 
 649       frame_complete = __ offset();
 650       address the_pc = __ pc();
 651 
 652       Label retaddr;
 653       __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
 654 
 655       __ mov(c_rarg0, rthread);
 656       __ mov(c_rarg1, rmethod);
 657       __ mov(c_rarg2, (int64_t)alloc_inline_receiver);
 658 
 659       __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::allocate_inline_types)));
 660       __ blr(rscratch1);
 661       __ bind(retaddr);
 662 
 663       oop_maps->add_gc_map(__ pc() - start, map);
 664       __ reset_last_Java_frame(false);
 665 
 666       reg_save.restore_live_registers(masm);





 667 
 668       Label no_exception;
 669       __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 670       __ cbz(rscratch1, no_exception);













 671 
 672       __ str(zr, Address(rthread, JavaThread::vm_result_oop_offset()));
 673       __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
 674       __ b(RuntimeAddress(StubRoutines::forward_exception_entry()));
 675 
 676       __ bind(no_exception);
 677 
 678       // We get an array of objects from the runtime call
 679       __ get_vm_result_oop(buf_array, rthread);
 680       __ get_vm_result_metadata(rmethod, rthread); // TODO: required to keep the callee Method live?
 681     }
 682   }








 683 
 684   // Since all args are passed on the stack, total_args_passed *
 685   // Interpreter::stackElementSize is the space we need.
 686 
 687   int total_args_passed = compute_total_args_passed_int(sig_extended);
 688   int extraspace = total_args_passed * Interpreter::stackElementSize;
 689 
 690   // stack is aligned, keep it that way
 691   extraspace = align_up(extraspace, StackAlignmentInBytes);
 692 
 693   // set senderSP value
 694   __ mov(r19_sender_sp, sp);
 695 
 696   __ sub(sp, sp, extraspace);
 697 
 698   // Now write the args into the outgoing interpreter space
 699 
 700   // next_arg_comp is the next argument from the compiler point of
 701   // view (inline type fields are passed in registers/on the stack). In
 702   // sig_extended, an inline type argument starts with: T_METADATA,
 703   // followed by the types of the fields of the inline type and T_VOID
 704   // to mark the end of the inline type. ignored counts the number of
 705   // T_METADATA/T_VOID. next_vt_arg is the next inline type argument:
 706   // used to get the buffer for that argument from the pool of buffers
 707   // we allocated above and want to pass to the
 708   // interpreter. next_arg_int is the next argument from the
 709   // interpreter point of view (inline types are passed by reference).
 710   for (int next_arg_comp = 0, ignored = 0, next_vt_arg = 0, next_arg_int = 0;
 711        next_arg_comp < sig_extended->length(); next_arg_comp++) {
 712     assert(ignored <= next_arg_comp, "shouldn't skip over more slots than there are arguments");
 713     assert(next_arg_int <= total_args_passed, "more arguments for the interpreter than expected?");
 714     BasicType bt = sig_extended->at(next_arg_comp)._bt;
 715     int st_off = (total_args_passed - next_arg_int - 1) * Interpreter::stackElementSize;
 716     if (!InlineTypePassFieldsAsArgs || bt != T_METADATA) {
 717       int next_off = st_off - Interpreter::stackElementSize;
 718       const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : st_off;
 719       const VMRegPair reg_pair = regs[next_arg_comp-ignored];
 720       size_t size_in_bytes = reg_pair.second()->is_valid() ? 8 : 4;
 721       gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
 722                              size_in_bytes, reg_pair, Address(sp, offset), tmp1, tmp2, tmp3, extraspace, false);
 723       next_arg_int++;
 724 #ifdef ASSERT
 725       if (bt == T_LONG || bt == T_DOUBLE) {
 726         // Overwrite the unused slot with known junk
 727         __ mov(rscratch1, CONST64(0xdeadffffdeadaaaa));
 728         __ str(rscratch1, Address(sp, st_off));



 729       }















 730 #endif /* ASSERT */
 731     } else {
 732       ignored++;
 733       // get the buffer from the just allocated pool of buffers
 734       int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + next_vt_arg * type2aelembytes(T_OBJECT);
 735       __ load_heap_oop(buf_oop, Address(buf_array, index), tmp1, tmp2);
 736       next_vt_arg++; next_arg_int++;
 737       int vt = 1;
 738       // write fields we get from compiled code in registers/stack
 739       // slots to the buffer: we know we are done with that inline type
 740       // argument when we hit the T_VOID that acts as an end of inline
 741       // type delimiter for this inline type. Inline types are flattened
 742       // so we might encounter embedded inline types. Each entry in
 743       // sig_extended contains a field offset in the buffer.
 744       Label L_null;
 745       do {
 746         next_arg_comp++;
 747         BasicType bt = sig_extended->at(next_arg_comp)._bt;
 748         BasicType prev_bt = sig_extended->at(next_arg_comp - 1)._bt;
 749         if (bt == T_METADATA) {
 750           vt++;
 751           ignored++;
 752         } else if (bt == T_VOID && prev_bt != T_LONG && prev_bt != T_DOUBLE) {
 753           vt--;
 754           ignored++;
 755         } else {
 756           int off = sig_extended->at(next_arg_comp)._offset;
 757           if (off == -1) {
 758             // Nullable inline type argument, emit null check
 759             VMReg reg = regs[next_arg_comp-ignored].first();
 760             Label L_notNull;
 761             if (reg->is_stack()) {
 762               int ld_off = reg->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
 763               __ ldrb(tmp1, Address(sp, ld_off));
 764               __ cbnz(tmp1, L_notNull);
 765             } else {
 766               __ cbnz(reg->as_Register(), L_notNull);
 767             }
 768             __ str(zr, Address(sp, st_off));
 769             __ b(L_null);
 770             __ bind(L_notNull);
 771             continue;
 772           }
 773           assert(off > 0, "offset in object should be positive");
 774           size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
 775           bool is_oop = is_reference_type(bt);
 776           gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
 777                                  size_in_bytes, regs[next_arg_comp-ignored], Address(buf_oop, off), tmp1, tmp2, tmp3, extraspace, is_oop);
 778         }
 779       } while (vt != 0);
 780       // pass the buffer to the interpreter
 781       __ str(buf_oop, Address(sp, st_off));
 782       __ bind(L_null);










 783     }
 784   }
 785 
 786   __ mov(esp, sp); // Interp expects args on caller's expression stack
 787 
 788   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));
 789   __ br(rscratch1);
 790 }
 791 
 792 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, int comp_args_on_stack, const GrowableArray<SigEntry>* sig, const VMRegPair *regs) {
 793 





 794 
 795   // Note: r19_sender_sp contains the senderSP on entry. We must
 796   // preserve it since we may do a i2c -> c2i transition if we lose a
 797   // race where compiled code goes non-entrant while we get args
 798   // ready.
 799 
 800   // Adapters are frameless.
 801 
 802   // An i2c adapter is frameless because the *caller* frame, which is
 803   // interpreted, routinely repairs its own esp (from
 804   // interpreter_frame_last_sp), even if a callee has modified the
 805   // stack pointer.  It also recalculates and aligns sp.
 806 
 807   // A c2i adapter is frameless because the *callee* frame, which is
 808   // interpreted, routinely repairs its caller's sp (from sender_sp,
 809   // which is set up via the senderSP register).
 810 
 811   // In other words, if *either* the caller or callee is interpreted, we can
 812   // get the stack pointer repaired after a call.
 813 
 814   // This is why c2i and i2c adapters cannot be indefinitely composed.
 815   // In particular, if a c2i adapter were to somehow call an i2c adapter,
 816   // both caller and callee would be compiled methods, and neither would
 817   // clean up the stack pointer changes performed by the two adapters.
 818   // If this happens, control eventually transfers back to the compiled
 819   // caller, but with an uncorrected stack, causing delayed havoc.
 820 
 821   // Cut-out for having no stack args.
 822   int comp_words_on_stack = 0;
 823   if (comp_args_on_stack) {
 824      comp_words_on_stack = align_up(comp_args_on_stack * VMRegImpl::stack_slot_size, wordSize) >> LogBytesPerWord;
 825      __ sub(rscratch1, sp, comp_words_on_stack * wordSize);
 826      __ andr(sp, rscratch1, -16);
 827   }
 828 
 829   // Will jump to the compiled code just as if compiled code was doing it.
 830   // Pre-load the register-jump target early, to schedule it better.
 831   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::from_compiled_inline_offset())));
 832 
 833 #if INCLUDE_JVMCI
 834   if (EnableJVMCI) {
 835     // check if this call should be routed towards a specific entry point
 836     __ ldr(rscratch2, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 837     Label no_alternative_target;
 838     __ cbz(rscratch2, no_alternative_target);
 839     __ mov(rscratch1, rscratch2);
 840     __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 841     __ bind(no_alternative_target);
 842   }
 843 #endif // INCLUDE_JVMCI
 844 
 845   int total_args_passed = sig->length();
 846 
 847   // Now generate the shuffle code.
 848   for (int i = 0; i < total_args_passed; i++) {
 849     BasicType bt = sig->at(i)._bt;
 850     if (bt == T_VOID) {
 851       assert(i > 0 && (sig->at(i - 1)._bt == T_LONG || sig->at(i - 1)._bt == T_DOUBLE), "missing half");
 852       continue;
 853     }
 854 
 855     // Pick up 0, 1 or 2 words from SP+offset.
 856     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(), "scrambled load targets?");
 857 


 858     // Load in argument order going down.
 859     int ld_off = (total_args_passed - i - 1) * Interpreter::stackElementSize;
 860     // Point to interpreter value (vs. tag)
 861     int next_off = ld_off - Interpreter::stackElementSize;
 862     //
 863     //
 864     //
 865     VMReg r_1 = regs[i].first();
 866     VMReg r_2 = regs[i].second();
 867     if (!r_1->is_valid()) {
 868       assert(!r_2->is_valid(), "");
 869       continue;
 870     }
 871     if (r_1->is_stack()) {
 872       // Convert stack slot to an SP offset (+ wordSize to account for return address )
 873       int st_off = regs[i].first()->reg2stack() * VMRegImpl::stack_slot_size;
 874       if (!r_2->is_valid()) {
 875         // sign extend???
 876         __ ldrsw(rscratch2, Address(esp, ld_off));
 877         __ str(rscratch2, Address(sp, st_off));
 878       } else {
 879         //
 880         // We are using two optoregs. This can be either T_OBJECT,
 881         // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 882         // two slots but only uses one for thr T_LONG or T_DOUBLE case
 883         // So we must adjust where to pick up the data to match the
 884         // interpreter.
 885         //
 886         // Interpreter local[n] == MSW, local[n+1] == LSW however locals
 887         // are accessed as negative so LSW is at LOW address
 888 
 889         // ld_off is MSW so get LSW
 890         const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : ld_off;

 891         __ ldr(rscratch2, Address(esp, offset));
 892         // st_off is LSW (i.e. reg.first())
 893          __ str(rscratch2, Address(sp, st_off));
 894        }
 895      } else if (r_1->is_Register()) {  // Register argument
 896        Register r = r_1->as_Register();
 897        if (r_2->is_valid()) {
 898          //
 899          // We are using two VMRegs. This can be either T_OBJECT,
 900          // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 901          // two slots but only uses one for thr T_LONG or T_DOUBLE case
 902          // So we must adjust where to pick up the data to match the
 903          // interpreter.
 904 
 905         const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : ld_off;
 906 
 907          // this can be a misaligned move
 908          __ ldr(r, Address(esp, offset));
 909        } else {
 910          // sign extend and use a full word?
 911          __ ldrw(r, Address(esp, ld_off));
 912        }
 913      } else {
 914        if (!r_2->is_valid()) {
 915          __ ldrs(r_1->as_FloatRegister(), Address(esp, ld_off));
 916        } else {
 917          __ ldrd(r_1->as_FloatRegister(), Address(esp, next_off));
 918        }
 919      }
 920    }
 921 

















 922 
 923   __ mov(rscratch2, rscratch1);
 924   __ push_cont_fastpath(rthread); // Set JavaThread::_cont_fastpath to the sp of the oldest interpreted frame we know about; kills rscratch1
 925   __ mov(rscratch1, rscratch2);
 926 
 927   // 6243940 We might end up in handle_wrong_method if
 928   // the callee is deoptimized as we race thru here. If that
 929   // happens we don't want to take a safepoint because the
 930   // caller frame will look interpreted and arguments are now
 931   // "compiled" so it is much better to make this transition
 932   // invisible to the stack walking code. Unfortunately if
 933   // we try and find the callee by normal means a safepoint
 934   // is possible. So we stash the desired callee in the thread
 935   // and the vm will find there should this case occur.
 936 
 937   __ str(rmethod, Address(rthread, JavaThread::callee_target_offset()));

 938   __ br(rscratch1);
 939 }
 940 
 941 static void gen_inline_cache_check(MacroAssembler *masm, Label& skip_fixup) {
 942   Register data = rscratch2;
 943   __ ic_check(1 /* end_alignment */);
 944   __ ldr(rmethod, Address(data, CompiledICData::speculated_method_offset()));
 945 
 946   // Method might have been compiled since the call site was patched to
 947   // interpreted; if that is the case treat it as a miss so we can get
 948   // the call site corrected.
 949   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 950   __ cbz(rscratch1, skip_fixup);
 951   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 952 }
 953 
 954 // ---------------------------------------------------------------
 955 void SharedRuntime::generate_i2c2i_adapters(MacroAssembler* masm,

 956                                             int comp_args_on_stack,
 957                                             const GrowableArray<SigEntry>* sig,
 958                                             const VMRegPair* regs,
 959                                             const GrowableArray<SigEntry>* sig_cc,
 960                                             const VMRegPair* regs_cc,
 961                                             const GrowableArray<SigEntry>* sig_cc_ro,
 962                                             const VMRegPair* regs_cc_ro,
 963                                             AdapterHandlerEntry* handler,
 964                                             AdapterBlob*& new_adapter,
 965                                             bool allocate_code_blob) {
 966   address i2c_entry = __ pc();
 967   gen_i2c_adapter(masm, comp_args_on_stack, sig, regs);








 968 
 969   // -------------------------------------------------------------------------
 970   // Generate a C2I adapter.  On entry we know rmethod holds the Method* during calls
 971   // to the interpreter.  The args start out packed in the compiled layout.  They
 972   // need to be unpacked into the interpreter layout.  This will almost always
 973   // require some stack space.  We grow the current (compiled) stack, then repack
 974   // the args.  We  finally end in a jump to the generic interpreter entry point.
 975   // On exit from the interpreter, the interpreter will restore our SP (lest the
 976   // compiled code, which relies solely on SP and not FP, get sick).
 977 
 978   address c2i_unverified_entry        = __ pc();
 979   address c2i_unverified_inline_entry = __ pc();
 980   Label skip_fixup;




 981 
 982   gen_inline_cache_check(masm, skip_fixup);




 983 
 984   OopMapSet* oop_maps = new OopMapSet();
 985   int frame_complete = CodeOffsets::frame_never_safe;
 986   int frame_size_in_words = 0;
 987 
 988   // Scalarized c2i adapter with non-scalarized receiver (i.e., don't pack receiver)
 989   address c2i_no_clinit_check_entry = nullptr;
 990   address c2i_inline_ro_entry = __ pc();
 991   if (regs_cc != regs_cc_ro) {
 992     // No class init barrier needed because method is guaranteed to be non-static
 993     gen_c2i_adapter(masm, sig_cc_ro, regs_cc_ro, /* requires_clinit_barrier = */ false, c2i_no_clinit_check_entry,
 994                     skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ false);
 995     skip_fixup.reset();
 996   }
 997 
 998   // Scalarized c2i adapter
 999   address c2i_entry        = __ pc();
1000   address c2i_inline_entry = __ pc();
1001   gen_c2i_adapter(masm, sig_cc, regs_cc, /* requires_clinit_barrier = */ true, c2i_no_clinit_check_entry,
1002                   skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ true);
1003 
1004   // Non-scalarized c2i adapter
1005   if (regs != regs_cc) {
1006     c2i_unverified_inline_entry = __ pc();
1007     Label inline_entry_skip_fixup;
1008     gen_inline_cache_check(masm, inline_entry_skip_fixup);
1009 
1010     c2i_inline_entry = __ pc();
1011     gen_c2i_adapter(masm, sig, regs, /* requires_clinit_barrier = */ true, c2i_no_clinit_check_entry,
1012                     inline_entry_skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ false);
1013   }
1014 
1015   // The c2i adapters might safepoint and trigger a GC. The caller must make sure that
1016   // the GC knows about the location of oop argument locations passed to the c2i adapter.
1017   if (allocate_code_blob) {
1018     bool caller_must_gc_arguments = (regs != regs_cc);
1019     new_adapter = AdapterBlob::create(masm->code(), frame_complete, frame_size_in_words, oop_maps, caller_must_gc_arguments);
1020   }
1021 
1022   handler->set_entry_points(i2c_entry, c2i_entry, c2i_inline_entry, c2i_inline_ro_entry, c2i_unverified_entry,
1023                             c2i_unverified_inline_entry, c2i_no_clinit_check_entry);
1024 }
1025 
1026 static int c_calling_convention_priv(const BasicType *sig_bt,
1027                                          VMRegPair *regs,
1028                                          int total_args_passed) {
1029 
1030 // We return the amount of VMRegImpl stack slots we need to reserve for all
1031 // the arguments NOT counting out_preserve_stack_slots.
1032 
1033     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
1034       c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5,  c_rarg6,  c_rarg7
1035     };
1036     static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
1037       c_farg0, c_farg1, c_farg2, c_farg3,
1038       c_farg4, c_farg5, c_farg6, c_farg7
1039     };
1040 
1041     uint int_args = 0;
1042     uint fp_args = 0;
1043     uint stk_args = 0; // inc by 2 each time

2040   if (method->is_synchronized()) {
2041     Label count;
2042     const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
2043 
2044     // Get the handle (the 2nd argument)
2045     __ mov(oop_handle_reg, c_rarg1);
2046 
2047     // Get address of the box
2048 
2049     __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
2050 
2051     // Load the oop from the handle
2052     __ ldr(obj_reg, Address(oop_handle_reg, 0));
2053 
2054     if (LockingMode == LM_MONITOR) {
2055       __ b(slow_path_lock);
2056     } else if (LockingMode == LM_LEGACY) {
2057       // Load (object->mark() | 1) into swap_reg %r0
2058       __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2059       __ orr(swap_reg, rscratch1, 1);
2060       if (EnableValhalla) {
2061         // Mask inline_type bit such that we go to the slow path if object is an inline type
2062         __ andr(swap_reg, swap_reg, ~((int) markWord::inline_type_bit_in_place));
2063       }
2064 
2065       // Save (object->mark() | 1) into BasicLock's displaced header
2066       __ str(swap_reg, Address(lock_reg, mark_word_offset));
2067 
2068       // src -> dest iff dest == r0 else r0 <- dest
2069       __ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, count, /*fallthrough*/nullptr);
2070 
2071       // Hmm should this move to the slow path code area???
2072 
2073       // Test if the oopMark is an obvious stack pointer, i.e.,
2074       //  1) (mark & 3) == 0, and
2075       //  2) sp <= mark < mark + os::pagesize()
2076       // These 3 tests can be done by evaluating the following
2077       // expression: ((mark - sp) & (3 - os::vm_page_size())),
2078       // assuming both stack pointer and pagesize have their
2079       // least significant 2 bits clear.
2080       // NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg
2081 
2082       __ sub(swap_reg, sp, swap_reg);
2083       __ neg(swap_reg, swap_reg);

3085 
3086   // exception pending => remove activation and forward to exception handler
3087 
3088   __ str(zr, Address(rthread, JavaThread::vm_result_oop_offset()));
3089 
3090   __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
3091   __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3092 
3093   // -------------
3094   // make sure all code is generated
3095   masm->flush();
3096 
3097   // return the  blob
3098   // frame_size_words or bytes??
3099   RuntimeStub* rs_blob = RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, true);
3100 
3101   AOTCodeCache::store_code_blob(*rs_blob, AOTCodeEntry::SharedBlob, (uint)id, name);
3102   return rs_blob;
3103 }
3104 
3105 BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(const InlineKlass* vk) {
3106   BufferBlob* buf = BufferBlob::create("inline types pack/unpack", 16 * K);
3107   CodeBuffer buffer(buf);
3108   short buffer_locs[20];
3109   buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
3110                                          sizeof(buffer_locs)/sizeof(relocInfo));
3111 
3112   MacroAssembler _masm(&buffer);
3113   MacroAssembler* masm = &_masm;
3114 
3115   const Array<SigEntry>* sig_vk = vk->extended_sig();
3116   const Array<VMRegPair>* regs = vk->return_regs();
3117 
3118   int pack_fields_jobject_off = __ offset();
3119   // Resolve pre-allocated buffer from JNI handle.
3120   // We cannot do this in generate_call_stub() because it requires GC code to be initialized.
3121   Register Rresult = r14;  // See StubGenerator::generate_call_stub().
3122   __ ldr(r0, Address(Rresult));
3123   __ resolve_jobject(r0 /* value */,
3124                      rthread /* thread */,
3125                      r12 /* tmp */);
3126   __ str(r0, Address(Rresult));
3127 
3128   int pack_fields_off = __ offset();
3129 
3130   int j = 1;
3131   for (int i = 0; i < sig_vk->length(); i++) {
3132     BasicType bt = sig_vk->at(i)._bt;
3133     if (bt == T_METADATA) {
3134       continue;
3135     }
3136     if (bt == T_VOID) {
3137       if (sig_vk->at(i-1)._bt == T_LONG ||
3138           sig_vk->at(i-1)._bt == T_DOUBLE) {
3139         j++;
3140       }
3141       continue;
3142     }
3143     int off = sig_vk->at(i)._offset;
3144     VMRegPair pair = regs->at(j);
3145     VMReg r_1 = pair.first();
3146     VMReg r_2 = pair.second();
3147     Address to(r0, off);
3148     if (bt == T_FLOAT) {
3149       __ strs(r_1->as_FloatRegister(), to);
3150     } else if (bt == T_DOUBLE) {
3151       __ strd(r_1->as_FloatRegister(), to);
3152     } else {
3153       Register val = r_1->as_Register();
3154       assert_different_registers(to.base(), val, r15, r16, r17);
3155       if (is_reference_type(bt)) {
3156         __ store_heap_oop(to, val, r15, r16, r17, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
3157       } else {
3158         __ store_sized_value(to, r_1->as_Register(), type2aelembytes(bt));
3159       }
3160     }
3161     j++;
3162   }
3163   assert(j == regs->length(), "missed a field?");
3164   if (vk->has_nullable_atomic_layout()) {
3165     // Zero the null marker (setting it to 1 would be better but would require an additional register)
3166     __ strb(zr, Address(r0, vk->null_marker_offset()));
3167   }
3168   __ ret(lr);
3169 
3170   int unpack_fields_off = __ offset();
3171 
3172   Label skip;
3173   Label not_null;
3174   __ cbnz(r0, not_null);
3175 
3176   // Return value is null. Zero oop registers to make the GC happy.
3177   j = 1;
3178   for (int i = 0; i < sig_vk->length(); i++) {
3179     BasicType bt = sig_vk->at(i)._bt;
3180     if (bt == T_METADATA) {
3181       continue;
3182     }
3183     if (bt == T_VOID) {
3184       if (sig_vk->at(i-1)._bt == T_LONG ||
3185           sig_vk->at(i-1)._bt == T_DOUBLE) {
3186         j++;
3187       }
3188       continue;
3189     }
3190     if (bt == T_OBJECT || bt == T_ARRAY) {
3191       VMRegPair pair = regs->at(j);
3192       VMReg r_1 = pair.first();
3193       __ mov(r_1->as_Register(), zr);
3194     }
3195     j++;
3196   }
3197   __ b(skip);
3198   __ bind(not_null);
3199 
3200   j = 1;
3201   for (int i = 0; i < sig_vk->length(); i++) {
3202     BasicType bt = sig_vk->at(i)._bt;
3203     if (bt == T_METADATA) {
3204       continue;
3205     }
3206     if (bt == T_VOID) {
3207       if (sig_vk->at(i-1)._bt == T_LONG ||
3208           sig_vk->at(i-1)._bt == T_DOUBLE) {
3209         j++;
3210       }
3211       continue;
3212     }
3213     int off = sig_vk->at(i)._offset;
3214     assert(off > 0, "offset in object should be positive");
3215     VMRegPair pair = regs->at(j);
3216     VMReg r_1 = pair.first();
3217     VMReg r_2 = pair.second();
3218     Address from(r0, off);
3219     if (bt == T_FLOAT) {
3220       __ ldrs(r_1->as_FloatRegister(), from);
3221     } else if (bt == T_DOUBLE) {
3222       __ ldrd(r_1->as_FloatRegister(), from);
3223     } else if (bt == T_OBJECT || bt == T_ARRAY) {
3224       assert_different_registers(r0, r_1->as_Register());
3225       __ load_heap_oop(r_1->as_Register(), from, rscratch1, rscratch2);
3226     } else {
3227       assert(is_java_primitive(bt), "unexpected basic type");
3228       assert_different_registers(r0, r_1->as_Register());
3229       size_t size_in_bytes = type2aelembytes(bt);
3230       __ load_sized_value(r_1->as_Register(), from, size_in_bytes, bt != T_CHAR && bt != T_BOOLEAN);
3231     }
3232     j++;
3233   }
3234   assert(j == regs->length(), "missed a field?");
3235 
3236   __ bind(skip);
3237 
3238   __ ret(lr);
3239 
3240   __ flush();
3241 
3242   return BufferedInlineTypeBlob::create(&buffer, pack_fields_off, pack_fields_jobject_off, unpack_fields_off);
3243 }
3244 
3245 // Continuation point for throwing of implicit exceptions that are
3246 // not handled in the current activation. Fabricates an exception
3247 // oop and initiates normal exception dispatching in this
3248 // frame. Since we need to preserve callee-saved values (currently
3249 // only for C2, but done for C1 as well) we need a callee-saved oop
3250 // map and therefore have to make these stubs into RuntimeStubs
3251 // rather than BufferBlobs.  If the compiler needs all registers to
3252 // be preserved between the fault point and the exception handler
3253 // then it must assume responsibility for that in
3254 // AbstractCompiler::continuation_for_implicit_null_exception or
3255 // continuation_for_implicit_division_by_zero_exception. All other
3256 // implicit exceptions (e.g., NullPointerException or
3257 // AbstractMethodError on entry) are either at call sites or
3258 // otherwise assume that stack unwinding will be initiated, so
3259 // caller saved registers were assumed volatile in the compiler.
3260 
3261 RuntimeStub* SharedRuntime::generate_throw_exception(SharedStubId id, address runtime_entry) {
3262   assert(is_throw_id(id), "expected a throw stub id");
3263 
3264   const char* name = SharedRuntime::stub_name(id);
< prev index next >