< prev index next >

src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp

Print this page

  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.hpp"
  29 #include "asm/macroAssembler.inline.hpp"

  30 #include "code/codeCache.hpp"
  31 #include "code/compiledIC.hpp"
  32 #include "code/debugInfoRec.hpp"
  33 #include "code/icBuffer.hpp"
  34 #include "code/vtableStubs.hpp"
  35 #include "compiler/oopMap.hpp"
  36 #include "gc/shared/barrierSetAssembler.hpp"
  37 #include "interpreter/interpreter.hpp"
  38 #include "interpreter/interp_masm.hpp"
  39 #include "logging/log.hpp"
  40 #include "memory/resourceArea.hpp"
  41 #include "nativeInst_aarch64.hpp"
  42 #include "oops/compiledICHolder.hpp"
  43 #include "oops/klass.inline.hpp"
  44 #include "oops/method.inline.hpp"
  45 #include "prims/methodHandles.hpp"
  46 #include "runtime/continuation.hpp"
  47 #include "runtime/continuationEntry.inline.hpp"
  48 #include "runtime/globals.hpp"
  49 #include "runtime/jniHandles.hpp"

 353       }
 354       break;
 355     case T_DOUBLE:
 356       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 357       if (fp_args < Argument::n_float_register_parameters_j) {
 358         regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
 359       } else {
 360         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 361         stk_args += 2;
 362       }
 363       break;
 364     default:
 365       ShouldNotReachHere();
 366       break;
 367     }
 368   }
 369 
 370   return align_up(stk_args, 2);
 371 }
 372 















































































 373 // Patch the callers callsite with entry to compiled code if it exists.
 374 static void patch_callers_callsite(MacroAssembler *masm) {
 375   Label L;
 376   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 377   __ cbz(rscratch1, L);
 378 
 379   __ enter();
 380   __ push_CPU_state();
 381 
 382   // VM needs caller's callsite
 383   // VM needs target method
 384   // This needs to be a long call since we will relocate this adapter to
 385   // the codeBuffer and it may not reach
 386 
 387 #ifndef PRODUCT
 388   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
 389 #endif
 390 
 391   __ mov(c_rarg0, rmethod);
 392   __ mov(c_rarg1, lr);
 393   __ authenticate_return_address(c_rarg1);
 394   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
 395   __ blr(rscratch1);
 396 
 397   // Explicit isb required because fixup_callers_callsite may change the code
 398   // stream.
 399   __ safepoint_isb();
 400 
 401   __ pop_CPU_state();
 402   // restore sp
 403   __ leave();
 404   __ bind(L);
 405 }
 406 












































































































 407 static void gen_c2i_adapter(MacroAssembler *masm,
 408                             int total_args_passed,
 409                             int comp_args_on_stack,
 410                             const BasicType *sig_bt,
 411                             const VMRegPair *regs,
 412                             Label& skip_fixup) {



























 413   // Before we get into the guts of the C2I adapter, see if we should be here
 414   // at all.  We've come from compiled code and are attempting to jump to the
 415   // interpreter, which means the caller made a static call to get here
 416   // (vcalls always get a compiled target if there is one).  Check for a
 417   // compiled target.  If there is one, we need to patch the caller's call.
 418   patch_callers_callsite(masm);
 419 
 420   __ bind(skip_fixup);
 421 
 422   int words_pushed = 0;





















 423 
 424   // Since all args are passed on the stack, total_args_passed *
 425   // Interpreter::stackElementSize is the space we need.
 426 
 427   int extraspace = total_args_passed * Interpreter::stackElementSize;

 428 
 429   __ mov(r19_sender_sp, sp);


 430 
 431   // stack is aligned, keep it that way
 432   extraspace = align_up(extraspace, 2*wordSize);

 433 
 434   if (extraspace)
 435     __ sub(sp, sp, extraspace);
 436 
 437   // Now write the args into the outgoing interpreter space
 438   for (int i = 0; i < total_args_passed; i++) {
 439     if (sig_bt[i] == T_VOID) {
 440       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
 441       continue;
 442     }
 443 
 444     // offset to start parameters
 445     int st_off   = (total_args_passed - i - 1) * Interpreter::stackElementSize;
 446     int next_off = st_off - Interpreter::stackElementSize;
 447 
 448     // Say 4 args:
 449     // i   st_off
 450     // 0   32 T_LONG
 451     // 1   24 T_VOID
 452     // 2   16 T_OBJECT
 453     // 3    8 T_BOOL
 454     // -    0 return address
 455     //
 456     // However to make thing extra confusing. Because we can fit a Java long/double in
 457     // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
 458     // leaves one slot empty and only stores to a single slot. In this case the
 459     // slot that is occupied is the T_VOID slot. See I said it was confusing.
 460 
 461     VMReg r_1 = regs[i].first();
 462     VMReg r_2 = regs[i].second();
 463     if (!r_1->is_valid()) {
 464       assert(!r_2->is_valid(), "");
 465       continue;




 466     }
 467     if (r_1->is_stack()) {
 468       // memory to memory use rscratch1
 469       int ld_off = (r_1->reg2stack() * VMRegImpl::stack_slot_size
 470                     + extraspace
 471                     + words_pushed * wordSize);
 472       if (!r_2->is_valid()) {
 473         // sign extend??
 474         __ ldrw(rscratch1, Address(sp, ld_off));
 475         __ str(rscratch1, Address(sp, st_off));
 476 
 477       } else {

 478 
 479         __ ldr(rscratch1, Address(sp, ld_off));




 480 
 481         // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
 482         // T_DOUBLE and T_LONG use two slots in the interpreter
 483         if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
 484           // ld_off == LSW, ld_off+wordSize == MSW
 485           // st_off == MSW, next_off == LSW
 486           __ str(rscratch1, Address(sp, next_off));

























 487 #ifdef ASSERT
 488           // Overwrite the unused slot with known junk
 489           __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaaaull);
 490           __ str(rscratch1, Address(sp, st_off));
 491 #endif /* ASSERT */
 492         } else {
 493           __ str(rscratch1, Address(sp, st_off));
 494         }
 495       }
 496     } else if (r_1->is_Register()) {
 497       Register r = r_1->as_Register();
 498       if (!r_2->is_valid()) {
 499         // must be only an int (or less ) so move only 32bits to slot
 500         // why not sign extend??
 501         __ str(r, Address(sp, st_off));
 502       } else {
 503         // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
 504         // T_DOUBLE and T_LONG use two slots in the interpreter
 505         if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
 506           // jlong/double in gpr
 507 #ifdef ASSERT
 508           // Overwrite the unused slot with known junk
 509           __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaabull);
 510           __ str(rscratch1, Address(sp, st_off));
 511 #endif /* ASSERT */
 512           __ str(r, Address(sp, next_off));























 513         } else {
 514           __ str(r, Address(sp, st_off));





















 515         }
 516       }
 517     } else {
 518       assert(r_1->is_FloatRegister(), "");
 519       if (!r_2->is_valid()) {
 520         // only a float use just part of the slot
 521         __ strs(r_1->as_FloatRegister(), Address(sp, st_off));
 522       } else {
 523 #ifdef ASSERT
 524         // Overwrite the unused slot with known junk
 525         __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaacull);
 526         __ str(rscratch1, Address(sp, st_off));
 527 #endif /* ASSERT */
 528         __ strd(r_1->as_FloatRegister(), Address(sp, next_off));
 529       }
 530     }
 531   }
 532 
 533   __ mov(esp, sp); // Interp expects args on caller's expression stack
 534 
 535   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));
 536   __ br(rscratch1);
 537 }
 538 

 539 
 540 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
 541                                     int total_args_passed,
 542                                     int comp_args_on_stack,
 543                                     const BasicType *sig_bt,
 544                                     const VMRegPair *regs) {
 545 
 546   // Note: r19_sender_sp contains the senderSP on entry. We must
 547   // preserve it since we may do a i2c -> c2i transition if we lose a
 548   // race where compiled code goes non-entrant while we get args
 549   // ready.
 550 
 551   // Adapters are frameless.
 552 
 553   // An i2c adapter is frameless because the *caller* frame, which is
 554   // interpreted, routinely repairs its own esp (from
 555   // interpreter_frame_last_sp), even if a callee has modified the
 556   // stack pointer.  It also recalculates and aligns sp.
 557 
 558   // A c2i adapter is frameless because the *callee* frame, which is
 559   // interpreted, routinely repairs its caller's sp (from sender_sp,
 560   // which is set up via the senderSP register).
 561 
 562   // In other words, if *either* the caller or callee is interpreted, we can
 563   // get the stack pointer repaired after a call.
 564 

 587       range_check(masm, rax, r11,
 588                   StubRoutines::initial_stubs_code()->code_begin(),
 589                   StubRoutines::initial_stubs_code()->code_end(),
 590                   L_ok);
 591     }
 592     if (StubRoutines::final_stubs_code() != nullptr) {
 593       range_check(masm, rax, r11,
 594                   StubRoutines::final_stubs_code()->code_begin(),
 595                   StubRoutines::final_stubs_code()->code_end(),
 596                   L_ok);
 597     }
 598     const char* msg = "i2c adapter must return to an interpreter frame";
 599     __ block_comment(msg);
 600     __ stop(msg);
 601     __ bind(L_ok);
 602     __ block_comment("} verify_i2ce ");
 603 #endif
 604   }
 605 
 606   // Cut-out for having no stack args.
 607   int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord;
 608   if (comp_args_on_stack) {
 609     __ sub(rscratch1, sp, comp_words_on_stack * wordSize);
 610     __ andr(sp, rscratch1, -16);

 611   }
 612 
 613   // Will jump to the compiled code just as if compiled code was doing it.
 614   // Pre-load the register-jump target early, to schedule it better.
 615   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::from_compiled_offset())));
 616 
 617 #if INCLUDE_JVMCI
 618   if (EnableJVMCI) {
 619     // check if this call should be routed towards a specific entry point
 620     __ ldr(rscratch2, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 621     Label no_alternative_target;
 622     __ cbz(rscratch2, no_alternative_target);
 623     __ mov(rscratch1, rscratch2);
 624     __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 625     __ bind(no_alternative_target);
 626   }
 627 #endif // INCLUDE_JVMCI
 628 


 629   // Now generate the shuffle code.
 630   for (int i = 0; i < total_args_passed; i++) {
 631     if (sig_bt[i] == T_VOID) {
 632       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");

 633       continue;
 634     }
 635 
 636     // Pick up 0, 1 or 2 words from SP+offset.

 637 
 638     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(),
 639             "scrambled load targets?");
 640     // Load in argument order going down.
 641     int ld_off = (total_args_passed - i - 1)*Interpreter::stackElementSize;
 642     // Point to interpreter value (vs. tag)
 643     int next_off = ld_off - Interpreter::stackElementSize;
 644     //
 645     //
 646     //
 647     VMReg r_1 = regs[i].first();
 648     VMReg r_2 = regs[i].second();
 649     if (!r_1->is_valid()) {
 650       assert(!r_2->is_valid(), "");
 651       continue;
 652     }
 653     if (r_1->is_stack()) {
 654       // Convert stack slot to an SP offset (+ wordSize to account for return address )
 655       int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size;
 656       if (!r_2->is_valid()) {
 657         // sign extend???
 658         __ ldrsw(rscratch2, Address(esp, ld_off));
 659         __ str(rscratch2, Address(sp, st_off));
 660       } else {
 661         //
 662         // We are using two optoregs. This can be either T_OBJECT,
 663         // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 664         // two slots but only uses one for thr T_LONG or T_DOUBLE case
 665         // So we must adjust where to pick up the data to match the
 666         // interpreter.
 667         //
 668         // Interpreter local[n] == MSW, local[n+1] == LSW however locals
 669         // are accessed as negative so LSW is at LOW address
 670 
 671         // ld_off is MSW so get LSW
 672         const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
 673                            next_off : ld_off;
 674         __ ldr(rscratch2, Address(esp, offset));
 675         // st_off is LSW (i.e. reg.first())
 676         __ str(rscratch2, Address(sp, st_off));
 677       }
 678     } else if (r_1->is_Register()) {  // Register argument
 679       Register r = r_1->as_Register();
 680       if (r_2->is_valid()) {
 681         //
 682         // We are using two VMRegs. This can be either T_OBJECT,
 683         // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 684         // two slots but only uses one for thr T_LONG or T_DOUBLE case
 685         // So we must adjust where to pick up the data to match the
 686         // interpreter.

















 687 
 688         const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
 689                            next_off : ld_off;
 690 
 691         // this can be a misaligned move
 692         __ ldr(r, Address(esp, offset));
 693       } else {
 694         // sign extend and use a full word?
 695         __ ldrw(r, Address(esp, ld_off));
 696       }
 697     } else {
 698       if (!r_2->is_valid()) {
 699         __ ldrs(r_1->as_FloatRegister(), Address(esp, ld_off));
 700       } else {
 701         __ ldrd(r_1->as_FloatRegister(), Address(esp, next_off));
 702       }
 703     }
 704   }
 705 
 706   __ mov(rscratch2, rscratch1);
 707   __ push_cont_fastpath(rthread); // Set JavaThread::_cont_fastpath to the sp of the oldest interpreted frame we know about; kills rscratch1
 708   __ mov(rscratch1, rscratch2);
 709 
 710   // 6243940 We might end up in handle_wrong_method if
 711   // the callee is deoptimized as we race thru here. If that
 712   // happens we don't want to take a safepoint because the
 713   // caller frame will look interpreted and arguments are now
 714   // "compiled" so it is much better to make this transition
 715   // invisible to the stack walking code. Unfortunately if
 716   // we try and find the callee by normal means a safepoint
 717   // is possible. So we stash the desired callee in the thread
 718   // and the vm will find there should this case occur.
 719 
 720   __ str(rmethod, Address(rthread, JavaThread::callee_target_offset()));
 721 
 722   __ br(rscratch1);
 723 }
 724 
 725 // ---------------------------------------------------------------
 726 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
 727                                                             int total_args_passed,
 728                                                             int comp_args_on_stack,
 729                                                             const BasicType *sig_bt,
 730                                                             const VMRegPair *regs,
 731                                                             AdapterFingerPrint* fingerprint) {
 732   address i2c_entry = __ pc();
 733 
 734   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
 735 
 736   address c2i_unverified_entry = __ pc();
 737   Label skip_fixup;
 738 
 739   Label ok;
 740 
 741   Register holder = rscratch2;
 742   Register receiver = j_rarg0;
 743   Register tmp = r10;  // A call-clobbered register not used for arg passing
 744 
 745   // -------------------------------------------------------------------------
 746   // Generate a C2I adapter.  On entry we know rmethod holds the Method* during calls
 747   // to the interpreter.  The args start out packed in the compiled layout.  They
 748   // need to be unpacked into the interpreter layout.  This will almost always
 749   // require some stack space.  We grow the current (compiled) stack, then repack
 750   // the args.  We  finally end in a jump to the generic interpreter entry point.
 751   // On exit from the interpreter, the interpreter will restore our SP (lest the
 752   // compiled code, which relies solely on SP and not FP, get sick).
 753 
 754   {
 755     __ block_comment("c2i_unverified_entry {");
 756     __ load_klass(rscratch1, receiver);
 757     __ ldr(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
 758     __ cmp(rscratch1, tmp);
 759     __ ldr(rmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
 760     __ br(Assembler::EQ, ok);
 761     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 762 
 763     __ bind(ok);
 764     // Method might have been compiled since the call site was patched to
 765     // interpreted; if that is the case treat it as a miss so we can get
 766     // the call site corrected.
 767     __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 768     __ cbz(rscratch1, skip_fixup);
 769     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 770     __ block_comment("} c2i_unverified_entry");
 771   }

 772 
 773   address c2i_entry = __ pc();
 774 
 775   // Class initialization barrier for static methods
 776   address c2i_no_clinit_check_entry = nullptr;
 777   if (VM_Version::supports_fast_class_init_checks()) {
 778     Label L_skip_barrier;








 779 
 780     { // Bypass the barrier for non-static methods
 781       __ ldrw(rscratch1, Address(rmethod, Method::access_flags_offset()));
 782       __ andsw(zr, rscratch1, JVM_ACC_STATIC);
 783       __ br(Assembler::EQ, L_skip_barrier); // non-static
 784     }
 785 
 786     __ load_method_holder(rscratch2, rmethod);
 787     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 788     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 789 
 790     __ bind(L_skip_barrier);
 791     c2i_no_clinit_check_entry = __ pc();












 792   }
 793 
 794   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 795   bs->c2i_entry_barrier(masm);














 796 
 797   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
 798 
 799   return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);







 800 }
 801 
 802 static int c_calling_convention_priv(const BasicType *sig_bt,
 803                                          VMRegPair *regs,
 804                                          int total_args_passed) {
 805 
 806 // We return the amount of VMRegImpl stack slots we need to reserve for all
 807 // the arguments NOT counting out_preserve_stack_slots.
 808 
 809     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
 810       c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5,  c_rarg6,  c_rarg7
 811     };
 812     static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
 813       c_farg0, c_farg1, c_farg2, c_farg3,
 814       c_farg4, c_farg5, c_farg6, c_farg7
 815     };
 816 
 817     uint int_args = 0;
 818     uint fp_args = 0;
 819     uint stk_args = 0; // inc by 2 each time

1766   if (method->is_synchronized()) {
1767     Label count;
1768     const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
1769 
1770     // Get the handle (the 2nd argument)
1771     __ mov(oop_handle_reg, c_rarg1);
1772 
1773     // Get address of the box
1774 
1775     __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
1776 
1777     // Load the oop from the handle
1778     __ ldr(obj_reg, Address(oop_handle_reg, 0));
1779 
1780     if (LockingMode == LM_MONITOR) {
1781       __ b(slow_path_lock);
1782     } else if (LockingMode == LM_LEGACY) {
1783       // Load (object->mark() | 1) into swap_reg %r0
1784       __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1785       __ orr(swap_reg, rscratch1, 1);




1786 
1787       // Save (object->mark() | 1) into BasicLock's displaced header
1788       __ str(swap_reg, Address(lock_reg, mark_word_offset));
1789 
1790       // src -> dest iff dest == r0 else r0 <- dest
1791       __ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, count, /*fallthrough*/nullptr);
1792 
1793       // Hmm should this move to the slow path code area???
1794 
1795       // Test if the oopMark is an obvious stack pointer, i.e.,
1796       //  1) (mark & 3) == 0, and
1797       //  2) sp <= mark < mark + os::pagesize()
1798       // These 3 tests can be done by evaluating the following
1799       // expression: ((mark - sp) & (3 - os::vm_page_size())),
1800       // assuming both stack pointer and pagesize have their
1801       // least significant 2 bits clear.
1802       // NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg
1803 
1804       __ sub(swap_reg, sp, swap_reg);
1805       __ neg(swap_reg, swap_reg);

3088   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
3089 #endif
3090   // Clear the exception oop so GC no longer processes it as a root.
3091   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
3092 
3093   // r0: exception oop
3094   // r8:  exception handler
3095   // r4: exception pc
3096   // Jump to handler
3097 
3098   __ br(r8);
3099 
3100   // Make sure all code is generated
3101   masm->flush();
3102 
3103   // Set exception blob
3104   _exception_blob =  ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
3105 }
3106 
3107 #endif // COMPILER2


















































































































  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.hpp"
  29 #include "asm/macroAssembler.inline.hpp"
  30 #include "classfile/symbolTable.hpp"
  31 #include "code/codeCache.hpp"
  32 #include "code/compiledIC.hpp"
  33 #include "code/debugInfoRec.hpp"
  34 #include "code/icBuffer.hpp"
  35 #include "code/vtableStubs.hpp"
  36 #include "compiler/oopMap.hpp"
  37 #include "gc/shared/barrierSetAssembler.hpp"
  38 #include "interpreter/interpreter.hpp"
  39 #include "interpreter/interp_masm.hpp"
  40 #include "logging/log.hpp"
  41 #include "memory/resourceArea.hpp"
  42 #include "nativeInst_aarch64.hpp"
  43 #include "oops/compiledICHolder.hpp"
  44 #include "oops/klass.inline.hpp"
  45 #include "oops/method.inline.hpp"
  46 #include "prims/methodHandles.hpp"
  47 #include "runtime/continuation.hpp"
  48 #include "runtime/continuationEntry.inline.hpp"
  49 #include "runtime/globals.hpp"
  50 #include "runtime/jniHandles.hpp"

 354       }
 355       break;
 356     case T_DOUBLE:
 357       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 358       if (fp_args < Argument::n_float_register_parameters_j) {
 359         regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
 360       } else {
 361         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 362         stk_args += 2;
 363       }
 364       break;
 365     default:
 366       ShouldNotReachHere();
 367       break;
 368     }
 369   }
 370 
 371   return align_up(stk_args, 2);
 372 }
 373 
 374 
 375 const uint SharedRuntime::java_return_convention_max_int = Argument::n_int_register_parameters_j;
 376 const uint SharedRuntime::java_return_convention_max_float = Argument::n_float_register_parameters_j;
 377 
 378 int SharedRuntime::java_return_convention(const BasicType *sig_bt, VMRegPair *regs, int total_args_passed) {
 379 
 380   // Create the mapping between argument positions and registers.
 381 
 382   static const Register INT_ArgReg[java_return_convention_max_int] = {
 383     r0 /* j_rarg7 */, j_rarg6, j_rarg5, j_rarg4, j_rarg3, j_rarg2, j_rarg1, j_rarg0
 384   };
 385 
 386   static const FloatRegister FP_ArgReg[java_return_convention_max_float] = {
 387     j_farg0, j_farg1, j_farg2, j_farg3, j_farg4, j_farg5, j_farg6, j_farg7
 388   };
 389 
 390   uint int_args = 0;
 391   uint fp_args = 0;
 392 
 393   for (int i = 0; i < total_args_passed; i++) {
 394     switch (sig_bt[i]) {
 395     case T_BOOLEAN:
 396     case T_CHAR:
 397     case T_BYTE:
 398     case T_SHORT:
 399     case T_INT:
 400       if (int_args < SharedRuntime::java_return_convention_max_int) {
 401         regs[i].set1(INT_ArgReg[int_args]->as_VMReg());
 402         int_args ++;
 403       } else {
 404         return -1;
 405       }
 406       break;
 407     case T_VOID:
 408       // halves of T_LONG or T_DOUBLE
 409       assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
 410       regs[i].set_bad();
 411       break;
 412     case T_LONG:
 413       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 414       // fall through
 415     case T_OBJECT:
 416     case T_ARRAY:
 417     case T_ADDRESS:
 418       // Should T_METADATA be added to java_calling_convention as well ?
 419     case T_METADATA:
 420       if (int_args < SharedRuntime::java_return_convention_max_int) {
 421         regs[i].set2(INT_ArgReg[int_args]->as_VMReg());
 422         int_args ++;
 423       } else {
 424         return -1;
 425       }
 426       break;
 427     case T_FLOAT:
 428       if (fp_args < SharedRuntime::java_return_convention_max_float) {
 429         regs[i].set1(FP_ArgReg[fp_args]->as_VMReg());
 430         fp_args ++;
 431       } else {
 432         return -1;
 433       }
 434       break;
 435     case T_DOUBLE:
 436       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 437       if (fp_args < SharedRuntime::java_return_convention_max_float) {
 438         regs[i].set2(FP_ArgReg[fp_args]->as_VMReg());
 439         fp_args ++;
 440       } else {
 441         return -1;
 442       }
 443       break;
 444     default:
 445       ShouldNotReachHere();
 446       break;
 447     }
 448   }
 449 
 450   return int_args + fp_args;
 451 }
 452 
 453 // Patch the callers callsite with entry to compiled code if it exists.
 454 static void patch_callers_callsite(MacroAssembler *masm) {
 455   Label L;
 456   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 457   __ cbz(rscratch1, L);
 458 
 459   __ enter();
 460   __ push_CPU_state();
 461 
 462   // VM needs caller's callsite
 463   // VM needs target method
 464   // This needs to be a long call since we will relocate this adapter to
 465   // the codeBuffer and it may not reach
 466 
 467 #ifndef PRODUCT
 468   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
 469 #endif
 470 
 471   __ mov(c_rarg0, rmethod);
 472   __ mov(c_rarg1, lr);
 473   __ authenticate_return_address(c_rarg1);
 474   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
 475   __ blr(rscratch1);
 476 
 477   // Explicit isb required because fixup_callers_callsite may change the code
 478   // stream.
 479   __ safepoint_isb();
 480 
 481   __ pop_CPU_state();
 482   // restore sp
 483   __ leave();
 484   __ bind(L);
 485 }
 486 
 487 // For each inline type argument, sig includes the list of fields of
 488 // the inline type. This utility function computes the number of
 489 // arguments for the call if inline types are passed by reference (the
 490 // calling convention the interpreter expects).
 491 static int compute_total_args_passed_int(const GrowableArray<SigEntry>* sig_extended) {
 492   int total_args_passed = 0;
 493   if (InlineTypePassFieldsAsArgs) {
 494      for (int i = 0; i < sig_extended->length(); i++) {
 495        BasicType bt = sig_extended->at(i)._bt;
 496        if (bt == T_METADATA) {
 497          // In sig_extended, an inline type argument starts with:
 498          // T_METADATA, followed by the types of the fields of the
 499          // inline type and T_VOID to mark the end of the value
 500          // type. Inline types are flattened so, for instance, in the
 501          // case of an inline type with an int field and an inline type
 502          // field that itself has 2 fields, an int and a long:
 503          // T_METADATA T_INT T_METADATA T_INT T_LONG T_VOID (second
 504          // slot for the T_LONG) T_VOID (inner inline type) T_VOID
 505          // (outer inline type)
 506          total_args_passed++;
 507          int vt = 1;
 508          do {
 509            i++;
 510            BasicType bt = sig_extended->at(i)._bt;
 511            BasicType prev_bt = sig_extended->at(i-1)._bt;
 512            if (bt == T_METADATA) {
 513              vt++;
 514            } else if (bt == T_VOID &&
 515                       prev_bt != T_LONG &&
 516                       prev_bt != T_DOUBLE) {
 517              vt--;
 518            }
 519          } while (vt != 0);
 520        } else {
 521          total_args_passed++;
 522        }
 523      }
 524   } else {
 525     total_args_passed = sig_extended->length();
 526   }
 527 
 528   return total_args_passed;
 529 }
 530 
 531 
 532 static void gen_c2i_adapter_helper(MacroAssembler* masm,
 533                                    BasicType bt,
 534                                    BasicType prev_bt,
 535                                    size_t size_in_bytes,
 536                                    const VMRegPair& reg_pair,
 537                                    const Address& to,
 538                                    Register tmp1,
 539                                    Register tmp2,
 540                                    Register tmp3,
 541                                    int extraspace,
 542                                    bool is_oop) {
 543   if (bt == T_VOID) {
 544     assert(prev_bt == T_LONG || prev_bt == T_DOUBLE, "missing half");
 545     return;
 546   }
 547 
 548   // Say 4 args:
 549   // i   st_off
 550   // 0   32 T_LONG
 551   // 1   24 T_VOID
 552   // 2   16 T_OBJECT
 553   // 3    8 T_BOOL
 554   // -    0 return address
 555   //
 556   // However to make thing extra confusing. Because we can fit a Java long/double in
 557   // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
 558   // leaves one slot empty and only stores to a single slot. In this case the
 559   // slot that is occupied is the T_VOID slot. See I said it was confusing.
 560 
 561   bool wide = (size_in_bytes == wordSize);
 562   VMReg r_1 = reg_pair.first();
 563   VMReg r_2 = reg_pair.second();
 564   assert(r_2->is_valid() == wide, "invalid size");
 565   if (!r_1->is_valid()) {
 566     assert(!r_2->is_valid(), "");
 567     return;
 568   }
 569 
 570   if (!r_1->is_FloatRegister()) {
 571     Register val = r25;
 572     if (r_1->is_stack()) {
 573       // memory to memory use r25 (scratch registers is used by store_heap_oop)
 574       int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
 575       __ load_sized_value(val, Address(sp, ld_off), size_in_bytes, /* is_signed */ false);
 576     } else {
 577       val = r_1->as_Register();
 578     }
 579     assert_different_registers(to.base(), val, tmp1, tmp2, tmp3);
 580     if (is_oop) {
 581       __ store_heap_oop(to, val, tmp1, tmp2, tmp3, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
 582     } else {
 583       __ store_sized_value(to, val, size_in_bytes);
 584     }
 585   } else {
 586     if (wide) {
 587       __ strd(r_1->as_FloatRegister(), to);
 588     } else {
 589       // only a float use just part of the slot
 590       __ strs(r_1->as_FloatRegister(), to);
 591     }
 592   }
 593 }
 594 
 595 static void gen_c2i_adapter(MacroAssembler *masm,
 596                             const GrowableArray<SigEntry>* sig_extended,


 597                             const VMRegPair *regs,
 598                             bool requires_clinit_barrier,
 599                             address& c2i_no_clinit_check_entry,
 600                             Label& skip_fixup,
 601                             address start,
 602                             OopMapSet* oop_maps,
 603                             int& frame_complete,
 604                             int& frame_size_in_words,
 605                             bool alloc_inline_receiver) {
 606   if (requires_clinit_barrier && VM_Version::supports_fast_class_init_checks()) {
 607     Label L_skip_barrier;
 608 
 609     { // Bypass the barrier for non-static methods
 610       __ ldrw(rscratch1, Address(rmethod, Method::access_flags_offset()));
 611       __ andsw(zr, rscratch1, JVM_ACC_STATIC);
 612       __ br(Assembler::EQ, L_skip_barrier); // non-static
 613     }
 614 
 615     __ load_method_holder(rscratch2, rmethod);
 616     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 617     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 618 
 619     __ bind(L_skip_barrier);
 620     c2i_no_clinit_check_entry = __ pc();
 621   }
 622 
 623   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 624   bs->c2i_entry_barrier(masm);
 625 
 626   // Before we get into the guts of the C2I adapter, see if we should be here
 627   // at all.  We've come from compiled code and are attempting to jump to the
 628   // interpreter, which means the caller made a static call to get here
 629   // (vcalls always get a compiled target if there is one).  Check for a
 630   // compiled target.  If there is one, we need to patch the caller's call.
 631   patch_callers_callsite(masm);
 632 
 633   __ bind(skip_fixup);
 634 
 635   // Name some registers to be used in the following code. We can use
 636   // anything except r0-r7 which are arguments in the Java calling
 637   // convention, rmethod (r12), and r13 which holds the outgoing sender
 638   // SP for the interpreter.
 639   Register buf_array = r10;   // Array of buffered inline types
 640   Register buf_oop = r11;     // Buffered inline type oop
 641   Register tmp1 = r15;
 642   Register tmp2 = r16;
 643   Register tmp3 = r17;
 644 
 645   if (InlineTypePassFieldsAsArgs) {
 646     // Is there an inline type argument?
 647     bool has_inline_argument = false;
 648     for (int i = 0; i < sig_extended->length() && !has_inline_argument; i++) {
 649       has_inline_argument = (sig_extended->at(i)._bt == T_METADATA);
 650     }
 651     if (has_inline_argument) {
 652       // There is at least an inline type argument: we're coming from
 653       // compiled code so we have no buffers to back the inline types
 654       // Allocate the buffers here with a runtime call.
 655       RegisterSaver reg_save(false /* save_vectors */);
 656       OopMap* map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
 657 
 658       frame_complete = __ offset();
 659       address the_pc = __ pc();
 660 
 661       Label retaddr;
 662       __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
 663 
 664       __ mov(c_rarg0, rthread);
 665       __ mov(c_rarg1, rmethod);
 666       __ mov(c_rarg2, (int64_t)alloc_inline_receiver);
 667 
 668       __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::allocate_inline_types)));
 669       __ blr(rscratch1);
 670       __ bind(retaddr);
 671 
 672       oop_maps->add_gc_map(__ pc() - start, map);
 673       __ reset_last_Java_frame(false);
 674 
 675       reg_save.restore_live_registers(masm);





 676 
 677       Label no_exception;
 678       __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 679       __ cbz(rscratch1, no_exception);













 680 
 681       __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
 682       __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
 683       __ b(RuntimeAddress(StubRoutines::forward_exception_entry()));
 684 
 685       __ bind(no_exception);
 686 
 687       // We get an array of objects from the runtime call
 688       __ get_vm_result(buf_array, rthread);
 689       __ get_vm_result_2(rmethod, rthread); // TODO: required to keep the callee Method live?
 690     }
 691   }








 692 
 693   // Since all args are passed on the stack, total_args_passed *
 694   // Interpreter::stackElementSize is the space we need.
 695 
 696   int total_args_passed = compute_total_args_passed_int(sig_extended);
 697   int extraspace = total_args_passed * Interpreter::stackElementSize;
 698 
 699   // stack is aligned, keep it that way
 700   extraspace = align_up(extraspace, StackAlignmentInBytes);
 701 
 702   // set senderSP value
 703   __ mov(r19_sender_sp, sp);
 704 
 705   __ sub(sp, sp, extraspace);
 706 
 707   // Now write the args into the outgoing interpreter space
 708 
 709   // next_arg_comp is the next argument from the compiler point of
 710   // view (inline type fields are passed in registers/on the stack). In
 711   // sig_extended, an inline type argument starts with: T_METADATA,
 712   // followed by the types of the fields of the inline type and T_VOID
 713   // to mark the end of the inline type. ignored counts the number of
 714   // T_METADATA/T_VOID. next_vt_arg is the next inline type argument:
 715   // used to get the buffer for that argument from the pool of buffers
 716   // we allocated above and want to pass to the
 717   // interpreter. next_arg_int is the next argument from the
 718   // interpreter point of view (inline types are passed by reference).
 719   for (int next_arg_comp = 0, ignored = 0, next_vt_arg = 0, next_arg_int = 0;
 720        next_arg_comp < sig_extended->length(); next_arg_comp++) {
 721     assert(ignored <= next_arg_comp, "shouldn't skip over more slots than there are arguments");
 722     assert(next_arg_int <= total_args_passed, "more arguments for the interpreter than expected?");
 723     BasicType bt = sig_extended->at(next_arg_comp)._bt;
 724     int st_off = (total_args_passed - next_arg_int - 1) * Interpreter::stackElementSize;
 725     if (!InlineTypePassFieldsAsArgs || bt != T_METADATA) {
 726       int next_off = st_off - Interpreter::stackElementSize;
 727       const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : st_off;
 728       const VMRegPair reg_pair = regs[next_arg_comp-ignored];
 729       size_t size_in_bytes = reg_pair.second()->is_valid() ? 8 : 4;
 730       gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
 731                              size_in_bytes, reg_pair, Address(sp, offset), tmp1, tmp2, tmp3, extraspace, false);
 732       next_arg_int++;
 733 #ifdef ASSERT
 734       if (bt == T_LONG || bt == T_DOUBLE) {
 735         // Overwrite the unused slot with known junk
 736         __ mov(rscratch1, CONST64(0xdeadffffdeadaaaa));
 737         __ str(rscratch1, Address(sp, st_off));



 738       }















 739 #endif /* ASSERT */
 740     } else {
 741       ignored++;
 742       // get the buffer from the just allocated pool of buffers
 743       int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + next_vt_arg * type2aelembytes(T_OBJECT);
 744       __ load_heap_oop(buf_oop, Address(buf_array, index), tmp1, tmp2);
 745       next_vt_arg++; next_arg_int++;
 746       int vt = 1;
 747       // write fields we get from compiled code in registers/stack
 748       // slots to the buffer: we know we are done with that inline type
 749       // argument when we hit the T_VOID that acts as an end of inline
 750       // type delimiter for this inline type. Inline types are flattened
 751       // so we might encounter embedded inline types. Each entry in
 752       // sig_extended contains a field offset in the buffer.
 753       Label L_null;
 754       do {
 755         next_arg_comp++;
 756         BasicType bt = sig_extended->at(next_arg_comp)._bt;
 757         BasicType prev_bt = sig_extended->at(next_arg_comp - 1)._bt;
 758         if (bt == T_METADATA) {
 759           vt++;
 760           ignored++;
 761         } else if (bt == T_VOID && prev_bt != T_LONG && prev_bt != T_DOUBLE) {
 762           vt--;
 763           ignored++;
 764         } else {
 765           int off = sig_extended->at(next_arg_comp)._offset;
 766           if (off == -1) {
 767             // Nullable inline type argument, emit null check
 768             VMReg reg = regs[next_arg_comp-ignored].first();
 769             Label L_notNull;
 770             if (reg->is_stack()) {
 771               int ld_off = reg->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
 772               __ ldrb(tmp1, Address(sp, ld_off));
 773               __ cbnz(tmp1, L_notNull);
 774             } else {
 775               __ cbnz(reg->as_Register(), L_notNull);
 776             }
 777             __ str(zr, Address(sp, st_off));
 778             __ b(L_null);
 779             __ bind(L_notNull);
 780             continue;
 781           }
 782           assert(off > 0, "offset in object should be positive");
 783           size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
 784           bool is_oop = is_reference_type(bt);
 785           gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
 786                                  size_in_bytes, regs[next_arg_comp-ignored], Address(buf_oop, off), tmp1, tmp2, tmp3, extraspace, is_oop);
 787         }
 788       } while (vt != 0);
 789       // pass the buffer to the interpreter
 790       __ str(buf_oop, Address(sp, st_off));
 791       __ bind(L_null);










 792     }
 793   }
 794 
 795   __ mov(esp, sp); // Interp expects args on caller's expression stack
 796 
 797   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));
 798   __ br(rscratch1);
 799 }
 800 
 801 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, int comp_args_on_stack, const GrowableArray<SigEntry>* sig, const VMRegPair *regs) {
 802 





 803 
 804   // Note: r19_sender_sp contains the senderSP on entry. We must
 805   // preserve it since we may do a i2c -> c2i transition if we lose a
 806   // race where compiled code goes non-entrant while we get args
 807   // ready.
 808 
 809   // Adapters are frameless.
 810 
 811   // An i2c adapter is frameless because the *caller* frame, which is
 812   // interpreted, routinely repairs its own esp (from
 813   // interpreter_frame_last_sp), even if a callee has modified the
 814   // stack pointer.  It also recalculates and aligns sp.
 815 
 816   // A c2i adapter is frameless because the *callee* frame, which is
 817   // interpreted, routinely repairs its caller's sp (from sender_sp,
 818   // which is set up via the senderSP register).
 819 
 820   // In other words, if *either* the caller or callee is interpreted, we can
 821   // get the stack pointer repaired after a call.
 822 

 845       range_check(masm, rax, r11,
 846                   StubRoutines::initial_stubs_code()->code_begin(),
 847                   StubRoutines::initial_stubs_code()->code_end(),
 848                   L_ok);
 849     }
 850     if (StubRoutines::final_stubs_code() != nullptr) {
 851       range_check(masm, rax, r11,
 852                   StubRoutines::final_stubs_code()->code_begin(),
 853                   StubRoutines::final_stubs_code()->code_end(),
 854                   L_ok);
 855     }
 856     const char* msg = "i2c adapter must return to an interpreter frame";
 857     __ block_comment(msg);
 858     __ stop(msg);
 859     __ bind(L_ok);
 860     __ block_comment("} verify_i2ce ");
 861 #endif
 862   }
 863 
 864   // Cut-out for having no stack args.
 865   int comp_words_on_stack = 0;
 866   if (comp_args_on_stack) {
 867      comp_words_on_stack = align_up(comp_args_on_stack * VMRegImpl::stack_slot_size, wordSize) >> LogBytesPerWord;
 868      __ sub(rscratch1, sp, comp_words_on_stack * wordSize);
 869      __ andr(sp, rscratch1, -16);
 870   }
 871 
 872   // Will jump to the compiled code just as if compiled code was doing it.
 873   // Pre-load the register-jump target early, to schedule it better.
 874   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::from_compiled_inline_offset())));
 875 
 876 #if INCLUDE_JVMCI
 877   if (EnableJVMCI) {
 878     // check if this call should be routed towards a specific entry point
 879     __ ldr(rscratch2, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 880     Label no_alternative_target;
 881     __ cbz(rscratch2, no_alternative_target);
 882     __ mov(rscratch1, rscratch2);
 883     __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 884     __ bind(no_alternative_target);
 885   }
 886 #endif // INCLUDE_JVMCI
 887 
 888   int total_args_passed = sig->length();
 889 
 890   // Now generate the shuffle code.
 891   for (int i = 0; i < total_args_passed; i++) {
 892     BasicType bt = sig->at(i)._bt;
 893     if (bt == T_VOID) {
 894       assert(i > 0 && (sig->at(i - 1)._bt == T_LONG || sig->at(i - 1)._bt == T_DOUBLE), "missing half");
 895       continue;
 896     }
 897 
 898     // Pick up 0, 1 or 2 words from SP+offset.
 899     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(), "scrambled load targets?");
 900 


 901     // Load in argument order going down.
 902     int ld_off = (total_args_passed - i - 1) * Interpreter::stackElementSize;
 903     // Point to interpreter value (vs. tag)
 904     int next_off = ld_off - Interpreter::stackElementSize;
 905     //
 906     //
 907     //
 908     VMReg r_1 = regs[i].first();
 909     VMReg r_2 = regs[i].second();
 910     if (!r_1->is_valid()) {
 911       assert(!r_2->is_valid(), "");
 912       continue;
 913     }
 914     if (r_1->is_stack()) {
 915       // Convert stack slot to an SP offset (+ wordSize to account for return address )
 916       int st_off = regs[i].first()->reg2stack() * VMRegImpl::stack_slot_size;
 917       if (!r_2->is_valid()) {
 918         // sign extend???
 919         __ ldrsw(rscratch2, Address(esp, ld_off));
 920         __ str(rscratch2, Address(sp, st_off));
 921       } else {
 922         //
 923         // We are using two optoregs. This can be either T_OBJECT,
 924         // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 925         // two slots but only uses one for thr T_LONG or T_DOUBLE case
 926         // So we must adjust where to pick up the data to match the
 927         // interpreter.
 928         //
 929         // Interpreter local[n] == MSW, local[n+1] == LSW however locals
 930         // are accessed as negative so LSW is at LOW address
 931 
 932         // ld_off is MSW so get LSW
 933         const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : ld_off;

 934         __ ldr(rscratch2, Address(esp, offset));
 935         // st_off is LSW (i.e. reg.first())
 936          __ str(rscratch2, Address(sp, st_off));
 937        }
 938      } else if (r_1->is_Register()) {  // Register argument
 939        Register r = r_1->as_Register();
 940        if (r_2->is_valid()) {
 941          //
 942          // We are using two VMRegs. This can be either T_OBJECT,
 943          // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 944          // two slots but only uses one for thr T_LONG or T_DOUBLE case
 945          // So we must adjust where to pick up the data to match the
 946          // interpreter.
 947 
 948         const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : ld_off;
 949 
 950          // this can be a misaligned move
 951          __ ldr(r, Address(esp, offset));
 952        } else {
 953          // sign extend and use a full word?
 954          __ ldrw(r, Address(esp, ld_off));
 955        }
 956      } else {
 957        if (!r_2->is_valid()) {
 958          __ ldrs(r_1->as_FloatRegister(), Address(esp, ld_off));
 959        } else {
 960          __ ldrd(r_1->as_FloatRegister(), Address(esp, next_off));
 961        }
 962      }
 963    }
 964 

















 965 
 966   __ mov(rscratch2, rscratch1);
 967   __ push_cont_fastpath(rthread); // Set JavaThread::_cont_fastpath to the sp of the oldest interpreted frame we know about; kills rscratch1
 968   __ mov(rscratch1, rscratch2);
 969 
 970   // 6243940 We might end up in handle_wrong_method if
 971   // the callee is deoptimized as we race thru here. If that
 972   // happens we don't want to take a safepoint because the
 973   // caller frame will look interpreted and arguments are now
 974   // "compiled" so it is much better to make this transition
 975   // invisible to the stack walking code. Unfortunately if
 976   // we try and find the callee by normal means a safepoint
 977   // is possible. So we stash the desired callee in the thread
 978   // and the vm will find there should this case occur.
 979 
 980   __ str(rmethod, Address(rthread, JavaThread::callee_target_offset()));

 981   __ br(rscratch1);
 982 }
 983 
 984 static void gen_inline_cache_check(MacroAssembler *masm, Label& skip_fixup) {












 985 
 986   Label ok;
 987 
 988   Register holder = rscratch2;
 989   Register receiver = j_rarg0;
 990   Register tmp = r10;  // A call-clobbered register not used for arg passing
 991 
 992   // -------------------------------------------------------------------------
 993   // Generate a C2I adapter.  On entry we know rmethod holds the Method* during calls
 994   // to the interpreter.  The args start out packed in the compiled layout.  They
 995   // need to be unpacked into the interpreter layout.  This will almost always
 996   // require some stack space.  We grow the current (compiled) stack, then repack
 997   // the args.  We  finally end in a jump to the generic interpreter entry point.
 998   // On exit from the interpreter, the interpreter will restore our SP (lest the
 999   // compiled code, which relies solely on SP and not FP, get sick).
1000 
1001   {
1002     __ block_comment("c2i_unverified_entry {");
1003     __ load_klass(rscratch1, receiver);
1004     __ ldr(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
1005     __ cmp(rscratch1, tmp);
1006     __ ldr(rmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
1007     __ br(Assembler::EQ, ok);
1008     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1009 
1010     __ bind(ok);
1011     // Method might have been compiled since the call site was patched to
1012     // interpreted; if that is the case treat it as a miss so we can get
1013     // the call site corrected.
1014     __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
1015     __ cbz(rscratch1, skip_fixup);
1016     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1017     __ block_comment("} c2i_unverified_entry");
1018   }
1019 }
1020 

1021 
1022 // ---------------------------------------------------------------
1023 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler* masm,
1024                                                             int comp_args_on_stack,
1025                                                             const GrowableArray<SigEntry>* sig,
1026                                                             const VMRegPair* regs,
1027                                                             const GrowableArray<SigEntry>* sig_cc,
1028                                                             const VMRegPair* regs_cc,
1029                                                             const GrowableArray<SigEntry>* sig_cc_ro,
1030                                                             const VMRegPair* regs_cc_ro,
1031                                                             AdapterFingerPrint* fingerprint,
1032                                                             AdapterBlob*& new_adapter,
1033                                                             bool allocate_code_blob) {
1034 
1035   address i2c_entry = __ pc();
1036   gen_i2c_adapter(masm, comp_args_on_stack, sig, regs);



1037 
1038   address c2i_unverified_entry        = __ pc();
1039   address c2i_unverified_inline_entry = __ pc();
1040   Label skip_fixup;
1041 
1042   gen_inline_cache_check(masm, skip_fixup);
1043 
1044   OopMapSet* oop_maps = new OopMapSet();
1045   int frame_complete = CodeOffsets::frame_never_safe;
1046   int frame_size_in_words = 0;
1047 
1048   // Scalarized c2i adapter with non-scalarized receiver (i.e., don't pack receiver)
1049   address c2i_no_clinit_check_entry = nullptr;
1050   address c2i_inline_ro_entry = __ pc();
1051   if (regs_cc != regs_cc_ro) {
1052     // No class init barrier needed because method is guaranteed to be non-static
1053     gen_c2i_adapter(masm, sig_cc_ro, regs_cc_ro, /* requires_clinit_barrier = */ false, c2i_no_clinit_check_entry,
1054                     skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ false);
1055     skip_fixup.reset();
1056   }
1057 
1058   // Scalarized c2i adapter
1059   address c2i_entry        = __ pc();
1060   address c2i_inline_entry = __ pc();
1061   gen_c2i_adapter(masm, sig_cc, regs_cc, /* requires_clinit_barrier = */ true, c2i_no_clinit_check_entry,
1062                   skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ true);
1063 
1064   // Non-scalarized c2i adapter
1065   if (regs != regs_cc) {
1066     c2i_unverified_inline_entry = __ pc();
1067     Label inline_entry_skip_fixup;
1068     gen_inline_cache_check(masm, inline_entry_skip_fixup);
1069 
1070     c2i_inline_entry = __ pc();
1071     gen_c2i_adapter(masm, sig, regs, /* requires_clinit_barrier = */ true, c2i_no_clinit_check_entry,
1072                     inline_entry_skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ false);
1073   }
1074 

1075 
1076   // The c2i adapter might safepoint and trigger a GC. The caller must make sure that
1077   // the GC knows about the location of oop argument locations passed to the c2i adapter.
1078   if (allocate_code_blob) {
1079     bool caller_must_gc_arguments = (regs != regs_cc);
1080     new_adapter = AdapterBlob::create(masm->code(), frame_complete, frame_size_in_words, oop_maps, caller_must_gc_arguments);
1081   }
1082 
1083   return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_inline_entry, c2i_inline_ro_entry, c2i_unverified_entry, c2i_unverified_inline_entry, c2i_no_clinit_check_entry);
1084 }
1085 
1086 static int c_calling_convention_priv(const BasicType *sig_bt,
1087                                          VMRegPair *regs,
1088                                          int total_args_passed) {
1089 
1090 // We return the amount of VMRegImpl stack slots we need to reserve for all
1091 // the arguments NOT counting out_preserve_stack_slots.
1092 
1093     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
1094       c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5,  c_rarg6,  c_rarg7
1095     };
1096     static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
1097       c_farg0, c_farg1, c_farg2, c_farg3,
1098       c_farg4, c_farg5, c_farg6, c_farg7
1099     };
1100 
1101     uint int_args = 0;
1102     uint fp_args = 0;
1103     uint stk_args = 0; // inc by 2 each time

2050   if (method->is_synchronized()) {
2051     Label count;
2052     const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
2053 
2054     // Get the handle (the 2nd argument)
2055     __ mov(oop_handle_reg, c_rarg1);
2056 
2057     // Get address of the box
2058 
2059     __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
2060 
2061     // Load the oop from the handle
2062     __ ldr(obj_reg, Address(oop_handle_reg, 0));
2063 
2064     if (LockingMode == LM_MONITOR) {
2065       __ b(slow_path_lock);
2066     } else if (LockingMode == LM_LEGACY) {
2067       // Load (object->mark() | 1) into swap_reg %r0
2068       __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2069       __ orr(swap_reg, rscratch1, 1);
2070       if (EnableValhalla) {
2071         // Mask inline_type bit such that we go to the slow path if object is an inline type
2072         __ andr(swap_reg, swap_reg, ~((int) markWord::inline_type_bit_in_place));
2073       }
2074 
2075       // Save (object->mark() | 1) into BasicLock's displaced header
2076       __ str(swap_reg, Address(lock_reg, mark_word_offset));
2077 
2078       // src -> dest iff dest == r0 else r0 <- dest
2079       __ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, count, /*fallthrough*/nullptr);
2080 
2081       // Hmm should this move to the slow path code area???
2082 
2083       // Test if the oopMark is an obvious stack pointer, i.e.,
2084       //  1) (mark & 3) == 0, and
2085       //  2) sp <= mark < mark + os::pagesize()
2086       // These 3 tests can be done by evaluating the following
2087       // expression: ((mark - sp) & (3 - os::vm_page_size())),
2088       // assuming both stack pointer and pagesize have their
2089       // least significant 2 bits clear.
2090       // NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg
2091 
2092       __ sub(swap_reg, sp, swap_reg);
2093       __ neg(swap_reg, swap_reg);

3376   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
3377 #endif
3378   // Clear the exception oop so GC no longer processes it as a root.
3379   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
3380 
3381   // r0: exception oop
3382   // r8:  exception handler
3383   // r4: exception pc
3384   // Jump to handler
3385 
3386   __ br(r8);
3387 
3388   // Make sure all code is generated
3389   masm->flush();
3390 
3391   // Set exception blob
3392   _exception_blob =  ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
3393 }
3394 
3395 #endif // COMPILER2
3396 
3397 BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(const InlineKlass* vk) {
3398   BufferBlob* buf = BufferBlob::create("inline types pack/unpack", 16 * K);
3399   CodeBuffer buffer(buf);
3400   short buffer_locs[20];
3401   buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
3402                                          sizeof(buffer_locs)/sizeof(relocInfo));
3403 
3404   MacroAssembler _masm(&buffer);
3405   MacroAssembler* masm = &_masm;
3406 
3407   const Array<SigEntry>* sig_vk = vk->extended_sig();
3408   const Array<VMRegPair>* regs = vk->return_regs();
3409 
3410   int pack_fields_jobject_off = __ offset();
3411   // Resolve pre-allocated buffer from JNI handle.
3412   // We cannot do this in generate_call_stub() because it requires GC code to be initialized.
3413   Register Rresult = r14;  // See StubGenerator::generate_call_stub().
3414   __ ldr(r0, Address(Rresult));
3415   __ resolve_jobject(r0 /* value */,
3416                      rthread /* thread */,
3417                      r12 /* tmp */);
3418   __ str(r0, Address(Rresult));
3419 
3420   int pack_fields_off = __ offset();
3421 
3422   int j = 1;
3423   for (int i = 0; i < sig_vk->length(); i++) {
3424     BasicType bt = sig_vk->at(i)._bt;
3425     if (bt == T_METADATA) {
3426       continue;
3427     }
3428     if (bt == T_VOID) {
3429       if (sig_vk->at(i-1)._bt == T_LONG ||
3430           sig_vk->at(i-1)._bt == T_DOUBLE) {
3431         j++;
3432       }
3433       continue;
3434     }
3435     int off = sig_vk->at(i)._offset;
3436     VMRegPair pair = regs->at(j);
3437     VMReg r_1 = pair.first();
3438     VMReg r_2 = pair.second();
3439     Address to(r0, off);
3440     if (bt == T_FLOAT) {
3441       __ strs(r_1->as_FloatRegister(), to);
3442     } else if (bt == T_DOUBLE) {
3443       __ strd(r_1->as_FloatRegister(), to);
3444     } else {
3445       Register val = r_1->as_Register();
3446       assert_different_registers(to.base(), val, r15, r16, r17);
3447       if (is_reference_type(bt)) {
3448         __ store_heap_oop(to, val, r15, r16, r17, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
3449       } else {
3450         __ store_sized_value(to, r_1->as_Register(), type2aelembytes(bt));
3451       }
3452     }
3453     j++;
3454   }
3455   assert(j == regs->length(), "missed a field?");
3456 
3457   __ ret(lr);
3458 
3459   int unpack_fields_off = __ offset();
3460 
3461   Label skip;
3462   __ cbz(r0, skip);
3463 
3464   j = 1;
3465   for (int i = 0; i < sig_vk->length(); i++) {
3466     BasicType bt = sig_vk->at(i)._bt;
3467     if (bt == T_METADATA) {
3468       continue;
3469     }
3470     if (bt == T_VOID) {
3471       if (sig_vk->at(i-1)._bt == T_LONG ||
3472           sig_vk->at(i-1)._bt == T_DOUBLE) {
3473         j++;
3474       }
3475       continue;
3476     }
3477     int off = sig_vk->at(i)._offset;
3478     assert(off > 0, "offset in object should be positive");
3479     VMRegPair pair = regs->at(j);
3480     VMReg r_1 = pair.first();
3481     VMReg r_2 = pair.second();
3482     Address from(r0, off);
3483     if (bt == T_FLOAT) {
3484       __ ldrs(r_1->as_FloatRegister(), from);
3485     } else if (bt == T_DOUBLE) {
3486       __ ldrd(r_1->as_FloatRegister(), from);
3487     } else if (bt == T_OBJECT || bt == T_ARRAY) {
3488       assert_different_registers(r0, r_1->as_Register());
3489       __ load_heap_oop(r_1->as_Register(), from, rscratch1, rscratch2);
3490     } else {
3491       assert(is_java_primitive(bt), "unexpected basic type");
3492       assert_different_registers(r0, r_1->as_Register());
3493 
3494       size_t size_in_bytes = type2aelembytes(bt);
3495       __ load_sized_value(r_1->as_Register(), from, size_in_bytes, bt != T_CHAR && bt != T_BOOLEAN);
3496     }
3497     j++;
3498   }
3499   assert(j == regs->length(), "missed a field?");
3500 
3501   __ bind(skip);
3502 
3503   __ ret(lr);
3504 
3505   __ flush();
3506 
3507   return BufferedInlineTypeBlob::create(&buffer, pack_fields_off, pack_fields_jobject_off, unpack_fields_off);
3508 }
< prev index next >