< prev index next >

src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp

Print this page

  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef _WINDOWS
  26 #include "alloca.h"
  27 #endif
  28 #include "asm/macroAssembler.hpp"
  29 #include "asm/macroAssembler.inline.hpp"

  30 #include "code/aotCodeCache.hpp"
  31 #include "code/compiledIC.hpp"
  32 #include "code/debugInfoRec.hpp"
  33 #include "code/nativeInst.hpp"
  34 #include "code/vtableStubs.hpp"
  35 #include "compiler/oopMap.hpp"
  36 #include "gc/shared/collectedHeap.hpp"
  37 #include "gc/shared/gcLocker.hpp"
  38 #include "gc/shared/barrierSet.hpp"
  39 #include "gc/shared/barrierSetAssembler.hpp"
  40 #include "interpreter/interpreter.hpp"
  41 #include "logging/log.hpp"
  42 #include "memory/resourceArea.hpp"
  43 #include "memory/universe.hpp"
  44 #include "oops/klass.inline.hpp"
  45 #include "oops/method.inline.hpp"
  46 #include "prims/methodHandles.hpp"
  47 #include "runtime/continuation.hpp"
  48 #include "runtime/continuationEntry.inline.hpp"
  49 #include "runtime/globals.hpp"

 617       break;
 618     case T_DOUBLE:
 619       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 620       if (fp_args < Argument::n_float_register_parameters_j) {
 621         regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
 622       } else {
 623         stk_args = align_up(stk_args, 2);
 624         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 625         stk_args += 2;
 626       }
 627       break;
 628     default:
 629       ShouldNotReachHere();
 630       break;
 631     }
 632   }
 633 
 634   return stk_args;
 635 }
 636 

















































































 637 // Patch the callers callsite with entry to compiled code if it exists.
 638 static void patch_callers_callsite(MacroAssembler *masm) {
 639   Label L;
 640   __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), NULL_WORD);
 641   __ jcc(Assembler::equal, L);
 642 
 643   // Save the current stack pointer
 644   __ mov(r13, rsp);
 645   // Schedule the branch target address early.
 646   // Call into the VM to patch the caller, then jump to compiled callee
 647   // rax isn't live so capture return address while we easily can
 648   __ movptr(rax, Address(rsp, 0));
 649 
 650   // align stack so push_CPU_state doesn't fault
 651   __ andptr(rsp, -(StackAlignmentInBytes));
 652   __ push_CPU_state();
 653   __ vzeroupper();
 654   // VM needs caller's callsite
 655   // VM needs target method
 656   // This needs to be a long call since we will relocate this adapter to

 659   // Allocate argument register save area
 660   if (frame::arg_reg_save_area_bytes != 0) {
 661     __ subptr(rsp, frame::arg_reg_save_area_bytes);
 662   }
 663   __ mov(c_rarg0, rbx);
 664   __ mov(c_rarg1, rax);
 665   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
 666 
 667   // De-allocate argument register save area
 668   if (frame::arg_reg_save_area_bytes != 0) {
 669     __ addptr(rsp, frame::arg_reg_save_area_bytes);
 670   }
 671 
 672   __ vzeroupper();
 673   __ pop_CPU_state();
 674   // restore sp
 675   __ mov(rsp, r13);
 676   __ bind(L);
 677 }
 678 










































































































 679 static void gen_c2i_adapter(MacroAssembler *masm,
 680                             int total_args_passed,
 681                             int comp_args_on_stack,
 682                             const BasicType *sig_bt,
 683                             const VMRegPair *regs,
 684                             Label& skip_fixup) {































 685   // Before we get into the guts of the C2I adapter, see if we should be here
 686   // at all.  We've come from compiled code and are attempting to jump to the
 687   // interpreter, which means the caller made a static call to get here
 688   // (vcalls always get a compiled target if there is one).  Check for a
 689   // compiled target.  If there is one, we need to patch the caller's call.
 690   patch_callers_callsite(masm);
 691 
 692   __ bind(skip_fixup);
 693 










































 694   // Since all args are passed on the stack, total_args_passed *
 695   // Interpreter::stackElementSize is the space we need.
 696 
 697   assert(total_args_passed >= 0, "total_args_passed is %d", total_args_passed);
 698 
 699   int extraspace = (total_args_passed * Interpreter::stackElementSize);
 700 
 701   // stack is aligned, keep it that way
 702   // This is not currently needed or enforced by the interpreter, but
 703   // we might as well conform to the ABI.
 704   extraspace = align_up(extraspace, 2*wordSize);
 705 
 706   // set senderSP value
 707   __ lea(r13, Address(rsp, wordSize));
 708 
 709 #ifdef ASSERT
 710   __ check_stack_alignment(r13, "sender stack not aligned");
 711 #endif
 712   if (extraspace > 0) {
 713     // Pop the return address
 714     __ pop(rax);
 715 
 716     __ subptr(rsp, extraspace);
 717 
 718     // Push the return address
 719     __ push(rax);
 720 
 721     // Account for the return address location since we store it first rather
 722     // than hold it in a register across all the shuffling
 723     extraspace += wordSize;
 724   }
 725 
 726 #ifdef ASSERT
 727   __ check_stack_alignment(rsp, "callee stack not aligned", wordSize, rax);
 728 #endif
 729 
 730   // Now write the args into the outgoing interpreter space
 731   for (int i = 0; i < total_args_passed; i++) {
 732     if (sig_bt[i] == T_VOID) {
 733       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
 734       continue;
 735     }
 736 
 737     // offset to start parameters
 738     int st_off   = (total_args_passed - i) * Interpreter::stackElementSize;
 739     int next_off = st_off - Interpreter::stackElementSize;
 740 
 741     // Say 4 args:
 742     // i   st_off
 743     // 0   32 T_LONG
 744     // 1   24 T_VOID
 745     // 2   16 T_OBJECT
 746     // 3    8 T_BOOL
 747     // -    0 return address
 748     //
 749     // However to make thing extra confusing. Because we can fit a long/double in
 750     // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
 751     // leaves one slot empty and only stores to a single slot. In this case the
 752     // slot that is occupied is the T_VOID slot. See I said it was confusing.
 753 
 754     VMReg r_1 = regs[i].first();
 755     VMReg r_2 = regs[i].second();
 756     if (!r_1->is_valid()) {
 757       assert(!r_2->is_valid(), "");
 758       continue;
 759     }
 760     if (r_1->is_stack()) {
 761       // memory to memory use rax
 762       int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
 763       if (!r_2->is_valid()) {
 764         // sign extend??
 765         __ movl(rax, Address(rsp, ld_off));
 766         __ movptr(Address(rsp, st_off), rax);
 767 
 768       } else {
 769 
 770         __ movq(rax, Address(rsp, ld_off));
 771 
 772         // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
 773         // T_DOUBLE and T_LONG use two slots in the interpreter
 774         if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
 775           // ld_off == LSW, ld_off+wordSize == MSW
 776           // st_off == MSW, next_off == LSW
 777           __ movq(Address(rsp, next_off), rax);














 778 #ifdef ASSERT
 779           // Overwrite the unused slot with known junk
 780           __ mov64(rax, CONST64(0xdeadffffdeadaaaa));
 781           __ movptr(Address(rsp, st_off), rax);
 782 #endif /* ASSERT */
 783         } else {
 784           __ movq(Address(rsp, st_off), rax);
 785         }
 786       }
 787     } else if (r_1->is_Register()) {
 788       Register r = r_1->as_Register();
 789       if (!r_2->is_valid()) {
 790         // must be only an int (or less ) so move only 32bits to slot
 791         // why not sign extend??
 792         __ movl(Address(rsp, st_off), r);
 793       } else {
 794         // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
 795         // T_DOUBLE and T_LONG use two slots in the interpreter
 796         if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
 797           // long/double in gpr
 798 #ifdef ASSERT
 799           // Overwrite the unused slot with known junk
 800           __ mov64(rax, CONST64(0xdeadffffdeadaaab));
 801           __ movptr(Address(rsp, st_off), rax);
 802 #endif /* ASSERT */
 803           __ movq(Address(rsp, next_off), r);

























 804         } else {
 805           __ movptr(Address(rsp, st_off), r);





















 806         }
 807       }
 808     } else {
 809       assert(r_1->is_XMMRegister(), "");
 810       if (!r_2->is_valid()) {
 811         // only a float use just part of the slot
 812         __ movflt(Address(rsp, st_off), r_1->as_XMMRegister());
 813       } else {
 814 #ifdef ASSERT
 815         // Overwrite the unused slot with known junk
 816         __ mov64(rax, CONST64(0xdeadffffdeadaaac));
 817         __ movptr(Address(rsp, st_off), rax);
 818 #endif /* ASSERT */
 819         __ movdbl(Address(rsp, next_off), r_1->as_XMMRegister());
 820       }
 821     }
 822   }
 823 
 824   // Schedule the branch target address early.
 825   __ movptr(rcx, Address(rbx, in_bytes(Method::interpreter_entry_offset())));
 826   __ jmp(rcx);
 827 }
 828 
 829 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
 830                                     int total_args_passed,
 831                                     int comp_args_on_stack,
 832                                     const BasicType *sig_bt,
 833                                     const VMRegPair *regs) {
 834 
 835   // Note: r13 contains the senderSP on entry. We must preserve it since
 836   // we may do a i2c -> c2i transition if we lose a race where compiled
 837   // code goes non-entrant while we get args ready.
 838   // In addition we use r13 to locate all the interpreter args as
 839   // we must align the stack to 16 bytes on an i2c entry else we
 840   // lose alignment we expect in all compiled code and register
 841   // save code can segv when fxsave instructions find improperly
 842   // aligned stack pointer.
 843 
 844   // Adapters can be frameless because they do not require the caller
 845   // to perform additional cleanup work, such as correcting the stack pointer.
 846   // An i2c adapter is frameless because the *caller* frame, which is interpreted,
 847   // routinely repairs its own stack pointer (from interpreter_frame_last_sp),
 848   // even if a callee has modified the stack pointer.
 849   // A c2i adapter is frameless because the *callee* frame, which is interpreted,
 850   // routinely repairs its caller's stack pointer (from sender_sp, which is set
 851   // up via the senderSP register).
 852   // In other words, if *either* the caller or callee is interpreted, we can

 868   // Convert 4-byte c2 stack slots to words.
 869   int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord;
 870 
 871   if (comp_args_on_stack) {
 872     __ subptr(rsp, comp_words_on_stack * wordSize);
 873   }
 874 
 875   // Ensure compiled code always sees stack at proper alignment
 876   __ andptr(rsp, -16);
 877 
 878   // push the return address and misalign the stack that youngest frame always sees
 879   // as far as the placement of the call instruction
 880   __ push(rax);
 881 
 882   // Put saved SP in another register
 883   const Register saved_sp = rax;
 884   __ movptr(saved_sp, r11);
 885 
 886   // Will jump to the compiled code just as if compiled code was doing it.
 887   // Pre-load the register-jump target early, to schedule it better.
 888   __ movptr(r11, Address(rbx, in_bytes(Method::from_compiled_offset())));
 889 
 890 #if INCLUDE_JVMCI
 891   if (EnableJVMCI) {
 892     // check if this call should be routed towards a specific entry point
 893     __ cmpptr(Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), 0);
 894     Label no_alternative_target;
 895     __ jcc(Assembler::equal, no_alternative_target);
 896     __ movptr(r11, Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 897     __ movptr(Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), 0);
 898     __ bind(no_alternative_target);
 899   }
 900 #endif // INCLUDE_JVMCI
 901 


 902   // Now generate the shuffle code.  Pick up all register args and move the
 903   // rest through the floating point stack top.
 904   for (int i = 0; i < total_args_passed; i++) {
 905     if (sig_bt[i] == T_VOID) {

 906       // Longs and doubles are passed in native word order, but misaligned
 907       // in the 32-bit build.
 908       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");

 909       continue;
 910     }
 911 
 912     // Pick up 0, 1 or 2 words from SP+offset.
 913 
 914     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(),
 915             "scrambled load targets?");
 916     // Load in argument order going down.
 917     int ld_off = (total_args_passed - i)*Interpreter::stackElementSize;
 918     // Point to interpreter value (vs. tag)
 919     int next_off = ld_off - Interpreter::stackElementSize;
 920     //
 921     //
 922     //
 923     VMReg r_1 = regs[i].first();
 924     VMReg r_2 = regs[i].second();
 925     if (!r_1->is_valid()) {
 926       assert(!r_2->is_valid(), "");
 927       continue;
 928     }

 930       // Convert stack slot to an SP offset (+ wordSize to account for return address )
 931       int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size + wordSize;
 932 
 933       // We can use r13 as a temp here because compiled code doesn't need r13 as an input
 934       // and if we end up going thru a c2i because of a miss a reasonable value of r13
 935       // will be generated.
 936       if (!r_2->is_valid()) {
 937         // sign extend???
 938         __ movl(r13, Address(saved_sp, ld_off));
 939         __ movptr(Address(rsp, st_off), r13);
 940       } else {
 941         //
 942         // We are using two optoregs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE
 943         // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case
 944         // So we must adjust where to pick up the data to match the interpreter.
 945         //
 946         // Interpreter local[n] == MSW, local[n+1] == LSW however locals
 947         // are accessed as negative so LSW is at LOW address
 948 
 949         // ld_off is MSW so get LSW
 950         const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
 951                            next_off : ld_off;
 952         __ movq(r13, Address(saved_sp, offset));
 953         // st_off is LSW (i.e. reg.first())
 954         __ movq(Address(rsp, st_off), r13);
 955       }
 956     } else if (r_1->is_Register()) {  // Register argument
 957       Register r = r_1->as_Register();
 958       assert(r != rax, "must be different");
 959       if (r_2->is_valid()) {
 960         //
 961         // We are using two VMRegs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE
 962         // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case
 963         // So we must adjust where to pick up the data to match the interpreter.
 964 
 965         const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
 966                            next_off : ld_off;
 967 
 968         // this can be a misaligned move
 969         __ movq(r, Address(saved_sp, offset));
 970       } else {
 971         // sign extend and use a full word?
 972         __ movl(r, Address(saved_sp, ld_off));
 973       }
 974     } else {
 975       if (!r_2->is_valid()) {
 976         __ movflt(r_1->as_XMMRegister(), Address(saved_sp, ld_off));
 977       } else {
 978         __ movdbl(r_1->as_XMMRegister(), Address(saved_sp, next_off));
 979       }
 980     }
 981   }
 982 
 983   __ push_cont_fastpath(); // Set JavaThread::_cont_fastpath to the sp of the oldest interpreted frame we know about
 984 
 985   // 6243940 We might end up in handle_wrong_method if
 986   // the callee is deoptimized as we race thru here. If that
 987   // happens we don't want to take a safepoint because the
 988   // caller frame will look interpreted and arguments are now
 989   // "compiled" so it is much better to make this transition
 990   // invisible to the stack walking code. Unfortunately if
 991   // we try and find the callee by normal means a safepoint
 992   // is possible. So we stash the desired callee in the thread
 993   // and the vm will find there should this case occur.
 994 
 995   __ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx);
 996 
 997   // put Method* where a c2i would expect should we end up there
 998   // only needed because eof c2 resolve stubs return Method* as a result in
 999   // rax
1000   __ mov(rax, rbx);
1001   __ jmp(r11);
1002 }
1003 













1004 // ---------------------------------------------------------------
1005 void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
1006                                             int total_args_passed,
1007                                             int comp_args_on_stack,
1008                                             const BasicType *sig_bt,
1009                                             const VMRegPair *regs,
1010                                             AdapterHandlerEntry* handler) {






1011   address i2c_entry = __ pc();
1012 
1013   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
1014 
1015   // -------------------------------------------------------------------------
1016   // Generate a C2I adapter.  On entry we know rbx holds the Method* during calls
1017   // to the interpreter.  The args start out packed in the compiled layout.  They
1018   // need to be unpacked into the interpreter layout.  This will almost always
1019   // require some stack space.  We grow the current (compiled) stack, then repack
1020   // the args.  We  finally end in a jump to the generic interpreter entry point.
1021   // On exit from the interpreter, the interpreter will restore our SP (lest the
1022   // compiled code, which relies solely on SP and not RBP, get sick).
1023 
1024   address c2i_unverified_entry = __ pc();

1025   Label skip_fixup;
1026 
1027   Register data = rax;
1028   Register receiver = j_rarg0;
1029   Register temp = rbx;
1030 
1031   {
1032     __ ic_check(1 /* end_alignment */);
1033     __ movptr(rbx, Address(data, CompiledICData::speculated_method_offset()));
1034     // Method might have been compiled since the call site was patched to
1035     // interpreted if that is the case treat it as a miss so we can get
1036     // the call site corrected.
1037     __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), NULL_WORD);
1038     __ jcc(Assembler::equal, skip_fixup);
1039     __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1040   }
1041 
1042   address c2i_entry = __ pc();
1043 
1044   // Class initialization barrier for static methods
1045   address c2i_no_clinit_check_entry = nullptr;
1046   if (VM_Version::supports_fast_class_init_checks()) {
1047     Label L_skip_barrier;
1048     Register method = rbx;
1049 
1050     { // Bypass the barrier for non-static methods
1051       Register flags = rscratch1;
1052       __ load_unsigned_short(flags, Address(method, Method::access_flags_offset()));
1053       __ testl(flags, JVM_ACC_STATIC);
1054       __ jcc(Assembler::zero, L_skip_barrier); // non-static
1055     }
1056 
1057     Register klass = rscratch1;
1058     __ load_method_holder(klass, method);
1059     __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
1060 
1061     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
1062 
1063     __ bind(L_skip_barrier);
1064     c2i_no_clinit_check_entry = __ pc();
1065   }
1066 
1067   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1068   bs->c2i_entry_barrier(masm);














1069 
1070   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);





1071 
1072   handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
1073   return;
1074 }
1075 
1076 int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
1077                                          VMRegPair *regs,
1078                                          int total_args_passed) {
1079 
1080 // We return the amount of VMRegImpl stack slots we need to reserve for all
1081 // the arguments NOT counting out_preserve_stack_slots.
1082 
1083 // NOTE: These arrays will have to change when c1 is ported
1084 #ifdef _WIN64
1085     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
1086       c_rarg0, c_rarg1, c_rarg2, c_rarg3
1087     };
1088     static const XMMRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
1089       c_farg0, c_farg1, c_farg2, c_farg3
1090     };
1091 #else
1092     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
1093       c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5

2185     const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
2186 
2187     // Get the handle (the 2nd argument)
2188     __ mov(oop_handle_reg, c_rarg1);
2189 
2190     // Get address of the box
2191 
2192     __ lea(lock_reg, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2193 
2194     // Load the oop from the handle
2195     __ movptr(obj_reg, Address(oop_handle_reg, 0));
2196 
2197     if (LockingMode == LM_MONITOR) {
2198       __ jmp(slow_path_lock);
2199     } else if (LockingMode == LM_LEGACY) {
2200       // Load immediate 1 into swap_reg %rax
2201       __ movl(swap_reg, 1);
2202 
2203       // Load (object->mark() | 1) into swap_reg %rax
2204       __ orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));




2205 
2206       // Save (object->mark() | 1) into BasicLock's displaced header
2207       __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2208 
2209       // src -> dest iff dest == rax else rax <- dest
2210       __ lock();
2211       __ cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2212       __ jcc(Assembler::equal, count_mon);
2213 
2214       // Hmm should this move to the slow path code area???
2215 
2216       // Test if the oopMark is an obvious stack pointer, i.e.,
2217       //  1) (mark & 3) == 0, and
2218       //  2) rsp <= mark < mark + os::pagesize()
2219       // These 3 tests can be done by evaluating the following
2220       // expression: ((mark - rsp) & (3 - os::vm_page_size())),
2221       // assuming both stack pointer and pagesize have their
2222       // least significant 2 bits clear.
2223       // NOTE: the oopMark is in swap_reg %rax as the result of cmpxchg
2224 

3612   julong *scratch = (julong *)alloca(total_allocation);
3613 
3614   // Local scratch arrays
3615   julong
3616     *a = scratch + 0 * longwords,
3617     *n = scratch + 1 * longwords,
3618     *m = scratch + 2 * longwords;
3619 
3620   reverse_words((julong *)a_ints, a, longwords);
3621   reverse_words((julong *)n_ints, n, longwords);
3622 
3623   if (len >= MONTGOMERY_SQUARING_THRESHOLD) {
3624     ::montgomery_square(a, n, m, (julong)inv, longwords);
3625   } else {
3626     ::montgomery_multiply(a, a, n, m, (julong)inv, longwords);
3627   }
3628 
3629   reverse_words(m, (julong *)m_ints, longwords);
3630 }
3631 










































































































































3632 #if INCLUDE_JFR
3633 
3634 // For c2: c_rarg0 is junk, call to runtime to write a checkpoint.
3635 // It returns a jobject handle to the event writer.
3636 // The handle is dereferenced and the return value is the event writer oop.
3637 RuntimeStub* SharedRuntime::generate_jfr_write_checkpoint() {
3638   enum layout {
3639     rbp_off,
3640     rbpH_off,
3641     return_off,
3642     return_off2,
3643     framesize // inclusive of return address
3644   };
3645 
3646   const char* name = SharedRuntime::stub_name(SharedStubId::jfr_write_checkpoint_id);
3647   CodeBuffer code(name, 1024, 64);
3648   MacroAssembler* masm = new MacroAssembler(&code);
3649   address start = __ pc();
3650 
3651   __ enter();

3704   __ reset_last_Java_frame(true);
3705 
3706   __ leave();
3707   __ ret(0);
3708 
3709   OopMapSet* oop_maps = new OopMapSet();
3710   OopMap* map = new OopMap(framesize, 1);
3711   oop_maps->add_gc_map(frame_complete, map);
3712 
3713   RuntimeStub* stub =
3714     RuntimeStub::new_runtime_stub(name,
3715                                   &code,
3716                                   frame_complete,
3717                                   (framesize >> (LogBytesPerWord - LogBytesPerInt)),
3718                                   oop_maps,
3719                                   false);
3720   return stub;
3721 }
3722 
3723 #endif // INCLUDE_JFR
3724 

  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef _WINDOWS
  26 #include "alloca.h"
  27 #endif
  28 #include "asm/macroAssembler.hpp"
  29 #include "asm/macroAssembler.inline.hpp"
  30 #include "classfile/symbolTable.hpp"
  31 #include "code/aotCodeCache.hpp"
  32 #include "code/compiledIC.hpp"
  33 #include "code/debugInfoRec.hpp"
  34 #include "code/nativeInst.hpp"
  35 #include "code/vtableStubs.hpp"
  36 #include "compiler/oopMap.hpp"
  37 #include "gc/shared/collectedHeap.hpp"
  38 #include "gc/shared/gcLocker.hpp"
  39 #include "gc/shared/barrierSet.hpp"
  40 #include "gc/shared/barrierSetAssembler.hpp"
  41 #include "interpreter/interpreter.hpp"
  42 #include "logging/log.hpp"
  43 #include "memory/resourceArea.hpp"
  44 #include "memory/universe.hpp"
  45 #include "oops/klass.inline.hpp"
  46 #include "oops/method.inline.hpp"
  47 #include "prims/methodHandles.hpp"
  48 #include "runtime/continuation.hpp"
  49 #include "runtime/continuationEntry.inline.hpp"
  50 #include "runtime/globals.hpp"

 618       break;
 619     case T_DOUBLE:
 620       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 621       if (fp_args < Argument::n_float_register_parameters_j) {
 622         regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
 623       } else {
 624         stk_args = align_up(stk_args, 2);
 625         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 626         stk_args += 2;
 627       }
 628       break;
 629     default:
 630       ShouldNotReachHere();
 631       break;
 632     }
 633   }
 634 
 635   return stk_args;
 636 }
 637 
 638 // Same as java_calling_convention() but for multiple return
 639 // values. There's no way to store them on the stack so if we don't
 640 // have enough registers, multiple values can't be returned.
 641 const uint SharedRuntime::java_return_convention_max_int = Argument::n_int_register_parameters_j+1;
 642 const uint SharedRuntime::java_return_convention_max_float = Argument::n_float_register_parameters_j;
 643 int SharedRuntime::java_return_convention(const BasicType *sig_bt,
 644                                           VMRegPair *regs,
 645                                           int total_args_passed) {
 646   // Create the mapping between argument positions and
 647   // registers.
 648   static const Register INT_ArgReg[java_return_convention_max_int] = {
 649     rax, j_rarg5, j_rarg4, j_rarg3, j_rarg2, j_rarg1, j_rarg0
 650   };
 651   static const XMMRegister FP_ArgReg[java_return_convention_max_float] = {
 652     j_farg0, j_farg1, j_farg2, j_farg3,
 653     j_farg4, j_farg5, j_farg6, j_farg7
 654   };
 655 
 656 
 657   uint int_args = 0;
 658   uint fp_args = 0;
 659 
 660   for (int i = 0; i < total_args_passed; i++) {
 661     switch (sig_bt[i]) {
 662     case T_BOOLEAN:
 663     case T_CHAR:
 664     case T_BYTE:
 665     case T_SHORT:
 666     case T_INT:
 667       if (int_args < Argument::n_int_register_parameters_j+1) {
 668         regs[i].set1(INT_ArgReg[int_args]->as_VMReg());
 669         int_args++;
 670       } else {
 671         return -1;
 672       }
 673       break;
 674     case T_VOID:
 675       // halves of T_LONG or T_DOUBLE
 676       assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
 677       regs[i].set_bad();
 678       break;
 679     case T_LONG:
 680       assert(sig_bt[i + 1] == T_VOID, "expecting half");
 681       // fall through
 682     case T_OBJECT:
 683     case T_ARRAY:
 684     case T_ADDRESS:
 685     case T_METADATA:
 686       if (int_args < Argument::n_int_register_parameters_j+1) {
 687         regs[i].set2(INT_ArgReg[int_args]->as_VMReg());
 688         int_args++;
 689       } else {
 690         return -1;
 691       }
 692       break;
 693     case T_FLOAT:
 694       if (fp_args < Argument::n_float_register_parameters_j) {
 695         regs[i].set1(FP_ArgReg[fp_args]->as_VMReg());
 696         fp_args++;
 697       } else {
 698         return -1;
 699       }
 700       break;
 701     case T_DOUBLE:
 702       assert(sig_bt[i + 1] == T_VOID, "expecting half");
 703       if (fp_args < Argument::n_float_register_parameters_j) {
 704         regs[i].set2(FP_ArgReg[fp_args]->as_VMReg());
 705         fp_args++;
 706       } else {
 707         return -1;
 708       }
 709       break;
 710     default:
 711       ShouldNotReachHere();
 712       break;
 713     }
 714   }
 715 
 716   return int_args + fp_args;
 717 }
 718 
 719 // Patch the callers callsite with entry to compiled code if it exists.
 720 static void patch_callers_callsite(MacroAssembler *masm) {
 721   Label L;
 722   __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), NULL_WORD);
 723   __ jcc(Assembler::equal, L);
 724 
 725   // Save the current stack pointer
 726   __ mov(r13, rsp);
 727   // Schedule the branch target address early.
 728   // Call into the VM to patch the caller, then jump to compiled callee
 729   // rax isn't live so capture return address while we easily can
 730   __ movptr(rax, Address(rsp, 0));
 731 
 732   // align stack so push_CPU_state doesn't fault
 733   __ andptr(rsp, -(StackAlignmentInBytes));
 734   __ push_CPU_state();
 735   __ vzeroupper();
 736   // VM needs caller's callsite
 737   // VM needs target method
 738   // This needs to be a long call since we will relocate this adapter to

 741   // Allocate argument register save area
 742   if (frame::arg_reg_save_area_bytes != 0) {
 743     __ subptr(rsp, frame::arg_reg_save_area_bytes);
 744   }
 745   __ mov(c_rarg0, rbx);
 746   __ mov(c_rarg1, rax);
 747   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
 748 
 749   // De-allocate argument register save area
 750   if (frame::arg_reg_save_area_bytes != 0) {
 751     __ addptr(rsp, frame::arg_reg_save_area_bytes);
 752   }
 753 
 754   __ vzeroupper();
 755   __ pop_CPU_state();
 756   // restore sp
 757   __ mov(rsp, r13);
 758   __ bind(L);
 759 }
 760 
 761 // For each inline type argument, sig includes the list of fields of
 762 // the inline type. This utility function computes the number of
 763 // arguments for the call if inline types are passed by reference (the
 764 // calling convention the interpreter expects).
 765 static int compute_total_args_passed_int(const GrowableArray<SigEntry>* sig_extended) {
 766   int total_args_passed = 0;
 767   if (InlineTypePassFieldsAsArgs) {
 768     for (int i = 0; i < sig_extended->length(); i++) {
 769       BasicType bt = sig_extended->at(i)._bt;
 770       if (bt == T_METADATA) {
 771         // In sig_extended, an inline type argument starts with:
 772         // T_METADATA, followed by the types of the fields of the
 773         // inline type and T_VOID to mark the end of the value
 774         // type. Inline types are flattened so, for instance, in the
 775         // case of an inline type with an int field and an inline type
 776         // field that itself has 2 fields, an int and a long:
 777         // T_METADATA T_INT T_METADATA T_INT T_LONG T_VOID (second
 778         // slot for the T_LONG) T_VOID (inner inline type) T_VOID
 779         // (outer inline type)
 780         total_args_passed++;
 781         int vt = 1;
 782         do {
 783           i++;
 784           BasicType bt = sig_extended->at(i)._bt;
 785           BasicType prev_bt = sig_extended->at(i-1)._bt;
 786           if (bt == T_METADATA) {
 787             vt++;
 788           } else if (bt == T_VOID &&
 789                      prev_bt != T_LONG &&
 790                      prev_bt != T_DOUBLE) {
 791             vt--;
 792           }
 793         } while (vt != 0);
 794       } else {
 795         total_args_passed++;
 796       }
 797     }
 798   } else {
 799     total_args_passed = sig_extended->length();
 800   }
 801   return total_args_passed;
 802 }
 803 
 804 
 805 static void gen_c2i_adapter_helper(MacroAssembler* masm,
 806                                    BasicType bt,
 807                                    BasicType prev_bt,
 808                                    size_t size_in_bytes,
 809                                    const VMRegPair& reg_pair,
 810                                    const Address& to,
 811                                    int extraspace,
 812                                    bool is_oop) {
 813   if (bt == T_VOID) {
 814     assert(prev_bt == T_LONG || prev_bt == T_DOUBLE, "missing half");
 815     return;
 816   }
 817 
 818   // Say 4 args:
 819   // i   st_off
 820   // 0   32 T_LONG
 821   // 1   24 T_VOID
 822   // 2   16 T_OBJECT
 823   // 3    8 T_BOOL
 824   // -    0 return address
 825   //
 826   // However to make thing extra confusing. Because we can fit a long/double in
 827   // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
 828   // leaves one slot empty and only stores to a single slot. In this case the
 829   // slot that is occupied is the T_VOID slot. See I said it was confusing.
 830 
 831   bool wide = (size_in_bytes == wordSize);
 832   VMReg r_1 = reg_pair.first();
 833   VMReg r_2 = reg_pair.second();
 834   assert(r_2->is_valid() == wide, "invalid size");
 835   if (!r_1->is_valid()) {
 836     assert(!r_2->is_valid(), "must be invalid");
 837     return;
 838   }
 839 
 840   if (!r_1->is_XMMRegister()) {
 841     Register val = rax;
 842     if (r_1->is_stack()) {
 843       int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
 844       __ load_sized_value(val, Address(rsp, ld_off), size_in_bytes, /* is_signed */ false);
 845     } else {
 846       val = r_1->as_Register();
 847     }
 848     assert_different_registers(to.base(), val, rscratch1);
 849     if (is_oop) {
 850       __ push(r13);
 851       __ push(rbx);
 852       __ store_heap_oop(to, val, rscratch1, r13, rbx, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
 853       __ pop(rbx);
 854       __ pop(r13);
 855     } else {
 856       __ store_sized_value(to, val, size_in_bytes);
 857     }
 858   } else {
 859     if (wide) {
 860       __ movdbl(to, r_1->as_XMMRegister());
 861     } else {
 862       __ movflt(to, r_1->as_XMMRegister());
 863     }
 864   }
 865 }
 866 
 867 static void gen_c2i_adapter(MacroAssembler *masm,
 868                             const GrowableArray<SigEntry>* sig_extended,


 869                             const VMRegPair *regs,
 870                             bool requires_clinit_barrier,
 871                             address& c2i_no_clinit_check_entry,
 872                             Label& skip_fixup,
 873                             address start,
 874                             OopMapSet* oop_maps,
 875                             int& frame_complete,
 876                             int& frame_size_in_words,
 877                             bool alloc_inline_receiver) {
 878   if (requires_clinit_barrier && VM_Version::supports_fast_class_init_checks()) {
 879     Label L_skip_barrier;
 880     Register method = rbx;
 881 
 882     { // Bypass the barrier for non-static methods
 883       Register flags = rscratch1;
 884       __ load_unsigned_short(flags, Address(method, Method::access_flags_offset()));
 885       __ testl(flags, JVM_ACC_STATIC);
 886       __ jcc(Assembler::zero, L_skip_barrier); // non-static
 887     }
 888 
 889     Register klass = rscratch1;
 890     __ load_method_holder(klass, method);
 891     __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
 892 
 893     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
 894 
 895     __ bind(L_skip_barrier);
 896     c2i_no_clinit_check_entry = __ pc();
 897   }
 898 
 899   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 900   bs->c2i_entry_barrier(masm);
 901 
 902   // Before we get into the guts of the C2I adapter, see if we should be here
 903   // at all.  We've come from compiled code and are attempting to jump to the
 904   // interpreter, which means the caller made a static call to get here
 905   // (vcalls always get a compiled target if there is one).  Check for a
 906   // compiled target.  If there is one, we need to patch the caller's call.
 907   patch_callers_callsite(masm);
 908 
 909   __ bind(skip_fixup);
 910 
 911   if (InlineTypePassFieldsAsArgs) {
 912     // Is there an inline type argument?
 913     bool has_inline_argument = false;
 914     for (int i = 0; i < sig_extended->length() && !has_inline_argument; i++) {
 915       has_inline_argument = (sig_extended->at(i)._bt == T_METADATA);
 916     }
 917     if (has_inline_argument) {
 918       // There is at least an inline type argument: we're coming from
 919       // compiled code so we have no buffers to back the inline types.
 920       // Allocate the buffers here with a runtime call.
 921       OopMap* map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_vectors*/ false);
 922 
 923       frame_complete = __ offset();
 924 
 925       __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1);
 926 
 927       __ mov(c_rarg0, r15_thread);
 928       __ mov(c_rarg1, rbx);
 929       __ mov64(c_rarg2, (int64_t)alloc_inline_receiver);
 930       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::allocate_inline_types)));
 931 
 932       oop_maps->add_gc_map((int)(__ pc() - start), map);
 933       __ reset_last_Java_frame(false);
 934 
 935       RegisterSaver::restore_live_registers(masm);
 936 
 937       Label no_exception;
 938       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD);
 939       __ jcc(Assembler::equal, no_exception);
 940 
 941       __ movptr(Address(r15_thread, JavaThread::vm_result_oop_offset()), NULL_WORD);
 942       __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
 943       __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 944 
 945       __ bind(no_exception);
 946 
 947       // We get an array of objects from the runtime call
 948       __ get_vm_result_oop(rscratch2); // Use rscratch2 (r11) as temporary because rscratch1 (r10) is trashed by movptr()
 949       __ get_vm_result_metadata(rbx); // TODO: required to keep the callee Method live?
 950     }
 951   }
 952 
 953   // Since all args are passed on the stack, total_args_passed *
 954   // Interpreter::stackElementSize is the space we need.
 955   int total_args_passed = compute_total_args_passed_int(sig_extended);
 956   assert(total_args_passed >= 0, "total_args_passed is %d", total_args_passed);
 957 
 958   int extraspace = (total_args_passed * Interpreter::stackElementSize);
 959 
 960   // stack is aligned, keep it that way
 961   // This is not currently needed or enforced by the interpreter, but
 962   // we might as well conform to the ABI.
 963   extraspace = align_up(extraspace, 2*wordSize);
 964 
 965   // set senderSP value
 966   __ lea(r13, Address(rsp, wordSize));
 967 
 968 #ifdef ASSERT
 969   __ check_stack_alignment(r13, "sender stack not aligned");
 970 #endif
 971   if (extraspace > 0) {
 972     // Pop the return address
 973     __ pop(rax);
 974 
 975     __ subptr(rsp, extraspace);
 976 
 977     // Push the return address
 978     __ push(rax);
 979 
 980     // Account for the return address location since we store it first rather
 981     // than hold it in a register across all the shuffling
 982     extraspace += wordSize;
 983   }
 984 
 985 #ifdef ASSERT
 986   __ check_stack_alignment(rsp, "callee stack not aligned", wordSize, rax);
 987 #endif
 988 
 989   // Now write the args into the outgoing interpreter space




































 990 
 991   // next_arg_comp is the next argument from the compiler point of
 992   // view (inline type fields are passed in registers/on the stack). In
 993   // sig_extended, an inline type argument starts with: T_METADATA,
 994   // followed by the types of the fields of the inline type and T_VOID
 995   // to mark the end of the inline type. ignored counts the number of
 996   // T_METADATA/T_VOID. next_vt_arg is the next inline type argument:
 997   // used to get the buffer for that argument from the pool of buffers
 998   // we allocated above and want to pass to the
 999   // interpreter. next_arg_int is the next argument from the
1000   // interpreter point of view (inline types are passed by reference).
1001   for (int next_arg_comp = 0, ignored = 0, next_vt_arg = 0, next_arg_int = 0;
1002        next_arg_comp < sig_extended->length(); next_arg_comp++) {
1003     assert(ignored <= next_arg_comp, "shouldn't skip over more slots than there are arguments");
1004     assert(next_arg_int <= total_args_passed, "more arguments for the interpreter than expected?");
1005     BasicType bt = sig_extended->at(next_arg_comp)._bt;
1006     int st_off = (total_args_passed - next_arg_int) * Interpreter::stackElementSize;
1007     if (!InlineTypePassFieldsAsArgs || bt != T_METADATA) {
1008       int next_off = st_off - Interpreter::stackElementSize;
1009       const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : st_off;
1010       const VMRegPair reg_pair = regs[next_arg_comp-ignored];
1011       size_t size_in_bytes = reg_pair.second()->is_valid() ? 8 : 4;
1012       gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
1013                              size_in_bytes, reg_pair, Address(rsp, offset), extraspace, false);
1014       next_arg_int++;
1015 #ifdef ASSERT
1016       if (bt == T_LONG || bt == T_DOUBLE) {
1017         // Overwrite the unused slot with known junk
1018         __ mov64(rax, CONST64(0xdeadffffdeadaaaa));
1019         __ movptr(Address(rsp, st_off), rax);



1020       }















1021 #endif /* ASSERT */
1022     } else {
1023       ignored++;
1024       // get the buffer from the just allocated pool of buffers
1025       int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + next_vt_arg * type2aelembytes(T_OBJECT);
1026       __ load_heap_oop(r14, Address(rscratch2, index));
1027       next_vt_arg++; next_arg_int++;
1028       int vt = 1;
1029       // write fields we get from compiled code in registers/stack
1030       // slots to the buffer: we know we are done with that inline type
1031       // argument when we hit the T_VOID that acts as an end of inline
1032       // type delimiter for this inline type. Inline types are flattened
1033       // so we might encounter embedded inline types. Each entry in
1034       // sig_extended contains a field offset in the buffer.
1035       Label L_null;
1036       do {
1037         next_arg_comp++;
1038         BasicType bt = sig_extended->at(next_arg_comp)._bt;
1039         BasicType prev_bt = sig_extended->at(next_arg_comp-1)._bt;
1040         if (bt == T_METADATA) {
1041           vt++;
1042           ignored++;
1043         } else if (bt == T_VOID &&
1044                    prev_bt != T_LONG &&
1045                    prev_bt != T_DOUBLE) {
1046           vt--;
1047           ignored++;
1048         } else {
1049           int off = sig_extended->at(next_arg_comp)._offset;
1050           if (off == -1) {
1051             // Nullable inline type argument, emit null check
1052             VMReg reg = regs[next_arg_comp-ignored].first();
1053             Label L_notNull;
1054             if (reg->is_stack()) {
1055               int ld_off = reg->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
1056               __ testb(Address(rsp, ld_off), 1);
1057             } else {
1058               __ testb(reg->as_Register(), 1);
1059             }
1060             __ jcc(Assembler::notZero, L_notNull);
1061             __ movptr(Address(rsp, st_off), 0);
1062             __ jmp(L_null);
1063             __ bind(L_notNull);
1064             continue;
1065           }
1066           assert(off > 0, "offset in object should be positive");
1067           size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
1068           bool is_oop = is_reference_type(bt);
1069           gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
1070                                  size_in_bytes, regs[next_arg_comp-ignored], Address(r14, off), extraspace, is_oop);
1071         }
1072       } while (vt != 0);
1073       // pass the buffer to the interpreter
1074       __ movptr(Address(rsp, st_off), r14);
1075       __ bind(L_null);










1076     }
1077   }
1078 
1079   // Schedule the branch target address early.
1080   __ movptr(rcx, Address(rbx, in_bytes(Method::interpreter_entry_offset())));
1081   __ jmp(rcx);
1082 }
1083 
1084 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,

1085                                     int comp_args_on_stack,
1086                                     const GrowableArray<SigEntry>* sig,
1087                                     const VMRegPair *regs) {
1088 
1089   // Note: r13 contains the senderSP on entry. We must preserve it since
1090   // we may do a i2c -> c2i transition if we lose a race where compiled
1091   // code goes non-entrant while we get args ready.
1092   // In addition we use r13 to locate all the interpreter args as
1093   // we must align the stack to 16 bytes on an i2c entry else we
1094   // lose alignment we expect in all compiled code and register
1095   // save code can segv when fxsave instructions find improperly
1096   // aligned stack pointer.
1097 
1098   // Adapters can be frameless because they do not require the caller
1099   // to perform additional cleanup work, such as correcting the stack pointer.
1100   // An i2c adapter is frameless because the *caller* frame, which is interpreted,
1101   // routinely repairs its own stack pointer (from interpreter_frame_last_sp),
1102   // even if a callee has modified the stack pointer.
1103   // A c2i adapter is frameless because the *callee* frame, which is interpreted,
1104   // routinely repairs its caller's stack pointer (from sender_sp, which is set
1105   // up via the senderSP register).
1106   // In other words, if *either* the caller or callee is interpreted, we can

1122   // Convert 4-byte c2 stack slots to words.
1123   int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord;
1124 
1125   if (comp_args_on_stack) {
1126     __ subptr(rsp, comp_words_on_stack * wordSize);
1127   }
1128 
1129   // Ensure compiled code always sees stack at proper alignment
1130   __ andptr(rsp, -16);
1131 
1132   // push the return address and misalign the stack that youngest frame always sees
1133   // as far as the placement of the call instruction
1134   __ push(rax);
1135 
1136   // Put saved SP in another register
1137   const Register saved_sp = rax;
1138   __ movptr(saved_sp, r11);
1139 
1140   // Will jump to the compiled code just as if compiled code was doing it.
1141   // Pre-load the register-jump target early, to schedule it better.
1142   __ movptr(r11, Address(rbx, in_bytes(Method::from_compiled_inline_offset())));
1143 
1144 #if INCLUDE_JVMCI
1145   if (EnableJVMCI) {
1146     // check if this call should be routed towards a specific entry point
1147     __ cmpptr(Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), 0);
1148     Label no_alternative_target;
1149     __ jcc(Assembler::equal, no_alternative_target);
1150     __ movptr(r11, Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
1151     __ movptr(Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), 0);
1152     __ bind(no_alternative_target);
1153   }
1154 #endif // INCLUDE_JVMCI
1155 
1156   int total_args_passed = sig->length();
1157 
1158   // Now generate the shuffle code.  Pick up all register args and move the
1159   // rest through the floating point stack top.
1160   for (int i = 0; i < total_args_passed; i++) {
1161     BasicType bt = sig->at(i)._bt;
1162     if (bt == T_VOID) {
1163       // Longs and doubles are passed in native word order, but misaligned
1164       // in the 32-bit build.
1165       BasicType prev_bt = (i > 0) ? sig->at(i-1)._bt : T_ILLEGAL;
1166       assert(i > 0 && (prev_bt == T_LONG || prev_bt == T_DOUBLE), "missing half");
1167       continue;
1168     }
1169 
1170     // Pick up 0, 1 or 2 words from SP+offset.
1171 
1172     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(),
1173             "scrambled load targets?");
1174     // Load in argument order going down.
1175     int ld_off = (total_args_passed - i)*Interpreter::stackElementSize;
1176     // Point to interpreter value (vs. tag)
1177     int next_off = ld_off - Interpreter::stackElementSize;
1178     //
1179     //
1180     //
1181     VMReg r_1 = regs[i].first();
1182     VMReg r_2 = regs[i].second();
1183     if (!r_1->is_valid()) {
1184       assert(!r_2->is_valid(), "");
1185       continue;
1186     }

1188       // Convert stack slot to an SP offset (+ wordSize to account for return address )
1189       int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size + wordSize;
1190 
1191       // We can use r13 as a temp here because compiled code doesn't need r13 as an input
1192       // and if we end up going thru a c2i because of a miss a reasonable value of r13
1193       // will be generated.
1194       if (!r_2->is_valid()) {
1195         // sign extend???
1196         __ movl(r13, Address(saved_sp, ld_off));
1197         __ movptr(Address(rsp, st_off), r13);
1198       } else {
1199         //
1200         // We are using two optoregs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE
1201         // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case
1202         // So we must adjust where to pick up the data to match the interpreter.
1203         //
1204         // Interpreter local[n] == MSW, local[n+1] == LSW however locals
1205         // are accessed as negative so LSW is at LOW address
1206 
1207         // ld_off is MSW so get LSW
1208         const int offset = (bt==T_LONG||bt==T_DOUBLE)?
1209                            next_off : ld_off;
1210         __ movq(r13, Address(saved_sp, offset));
1211         // st_off is LSW (i.e. reg.first())
1212         __ movq(Address(rsp, st_off), r13);
1213       }
1214     } else if (r_1->is_Register()) {  // Register argument
1215       Register r = r_1->as_Register();
1216       assert(r != rax, "must be different");
1217       if (r_2->is_valid()) {
1218         //
1219         // We are using two VMRegs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE
1220         // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case
1221         // So we must adjust where to pick up the data to match the interpreter.
1222 
1223         const int offset = (bt==T_LONG||bt==T_DOUBLE)?
1224                            next_off : ld_off;
1225 
1226         // this can be a misaligned move
1227         __ movq(r, Address(saved_sp, offset));
1228       } else {
1229         // sign extend and use a full word?
1230         __ movl(r, Address(saved_sp, ld_off));
1231       }
1232     } else {
1233       if (!r_2->is_valid()) {
1234         __ movflt(r_1->as_XMMRegister(), Address(saved_sp, ld_off));
1235       } else {
1236         __ movdbl(r_1->as_XMMRegister(), Address(saved_sp, next_off));
1237       }
1238     }
1239   }
1240 
1241   __ push_cont_fastpath(); // Set JavaThread::_cont_fastpath to the sp of the oldest interpreted frame we know about
1242 
1243   // 6243940 We might end up in handle_wrong_method if
1244   // the callee is deoptimized as we race thru here. If that
1245   // happens we don't want to take a safepoint because the
1246   // caller frame will look interpreted and arguments are now
1247   // "compiled" so it is much better to make this transition
1248   // invisible to the stack walking code. Unfortunately if
1249   // we try and find the callee by normal means a safepoint
1250   // is possible. So we stash the desired callee in the thread
1251   // and the vm will find there should this case occur.
1252 
1253   __ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx);
1254 
1255   // put Method* where a c2i would expect should we end up there
1256   // only needed because of c2 resolve stubs return Method* as a result in
1257   // rax
1258   __ mov(rax, rbx);
1259   __ jmp(r11);
1260 }
1261 
1262 static void gen_inline_cache_check(MacroAssembler *masm, Label& skip_fixup) {
1263   Register data = rax;
1264   __ ic_check(1 /* end_alignment */);
1265   __ movptr(rbx, Address(data, CompiledICData::speculated_method_offset()));
1266 
1267   // Method might have been compiled since the call site was patched to
1268   // interpreted if that is the case treat it as a miss so we can get
1269   // the call site corrected.
1270   __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), NULL_WORD);
1271   __ jcc(Assembler::equal, skip_fixup);
1272   __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1273 }
1274 
1275 // ---------------------------------------------------------------
1276 void SharedRuntime::generate_i2c2i_adapters(MacroAssembler* masm,

1277                                             int comp_args_on_stack,
1278                                             const GrowableArray<SigEntry>* sig,
1279                                             const VMRegPair* regs,
1280                                             const GrowableArray<SigEntry>* sig_cc,
1281                                             const VMRegPair* regs_cc,
1282                                             const GrowableArray<SigEntry>* sig_cc_ro,
1283                                             const VMRegPair* regs_cc_ro,
1284                                             AdapterHandlerEntry* handler,
1285                                             AdapterBlob*& new_adapter,
1286                                             bool allocate_code_blob) {
1287   address i2c_entry = __ pc();
1288   gen_i2c_adapter(masm, comp_args_on_stack, sig, regs);

1289 
1290   // -------------------------------------------------------------------------
1291   // Generate a C2I adapter.  On entry we know rbx holds the Method* during calls
1292   // to the interpreter.  The args start out packed in the compiled layout.  They
1293   // need to be unpacked into the interpreter layout.  This will almost always
1294   // require some stack space.  We grow the current (compiled) stack, then repack
1295   // the args.  We  finally end in a jump to the generic interpreter entry point.
1296   // On exit from the interpreter, the interpreter will restore our SP (lest the
1297   // compiled code, which relies solely on SP and not RBP, get sick).
1298 
1299   address c2i_unverified_entry        = __ pc();
1300   address c2i_unverified_inline_entry = __ pc();
1301   Label skip_fixup;
1302 
1303   gen_inline_cache_check(masm, skip_fixup);


1304 
1305   OopMapSet* oop_maps = new OopMapSet();
1306   int frame_complete = CodeOffsets::frame_never_safe;
1307   int frame_size_in_words = 0;









1308 
1309   // Scalarized c2i adapter with non-scalarized receiver (i.e., don't pack receiver)
1310   address c2i_no_clinit_check_entry = nullptr;
1311   address c2i_inline_ro_entry = __ pc();
1312   if (regs_cc != regs_cc_ro) {
1313     // No class init barrier needed because method is guaranteed to be non-static
1314     gen_c2i_adapter(masm, sig_cc_ro, regs_cc_ro, /* requires_clinit_barrier = */ false, c2i_no_clinit_check_entry,
1315                     skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ false);
1316     skip_fixup.reset();













1317   }
1318 
1319   // Scalarized c2i adapter
1320   address c2i_entry        = __ pc();
1321   address c2i_inline_entry = __ pc();
1322   gen_c2i_adapter(masm, sig_cc, regs_cc, /* requires_clinit_barrier = */ true, c2i_no_clinit_check_entry,
1323                   skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ true);
1324 
1325   // Non-scalarized c2i adapter
1326   if (regs != regs_cc) {
1327     c2i_unverified_inline_entry = __ pc();
1328     Label inline_entry_skip_fixup;
1329     gen_inline_cache_check(masm, inline_entry_skip_fixup);
1330 
1331     c2i_inline_entry = __ pc();
1332     gen_c2i_adapter(masm, sig, regs, /* requires_clinit_barrier = */ true, c2i_no_clinit_check_entry,
1333                     inline_entry_skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ false);
1334   }
1335 
1336   // The c2i adapters might safepoint and trigger a GC. The caller must make sure that
1337   // the GC knows about the location of oop argument locations passed to the c2i adapter.
1338   if (allocate_code_blob) {
1339     bool caller_must_gc_arguments = (regs != regs_cc);
1340     new_adapter = AdapterBlob::create(masm->code(), frame_complete, frame_size_in_words, oop_maps, caller_must_gc_arguments);
1341   }
1342 
1343   handler->set_entry_points(i2c_entry, c2i_entry, c2i_inline_entry, c2i_inline_ro_entry, c2i_unverified_entry,
1344                             c2i_unverified_inline_entry, c2i_no_clinit_check_entry);
1345 }
1346 
1347 int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
1348                                          VMRegPair *regs,
1349                                          int total_args_passed) {
1350 
1351 // We return the amount of VMRegImpl stack slots we need to reserve for all
1352 // the arguments NOT counting out_preserve_stack_slots.
1353 
1354 // NOTE: These arrays will have to change when c1 is ported
1355 #ifdef _WIN64
1356     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
1357       c_rarg0, c_rarg1, c_rarg2, c_rarg3
1358     };
1359     static const XMMRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
1360       c_farg0, c_farg1, c_farg2, c_farg3
1361     };
1362 #else
1363     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
1364       c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5

2456     const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
2457 
2458     // Get the handle (the 2nd argument)
2459     __ mov(oop_handle_reg, c_rarg1);
2460 
2461     // Get address of the box
2462 
2463     __ lea(lock_reg, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2464 
2465     // Load the oop from the handle
2466     __ movptr(obj_reg, Address(oop_handle_reg, 0));
2467 
2468     if (LockingMode == LM_MONITOR) {
2469       __ jmp(slow_path_lock);
2470     } else if (LockingMode == LM_LEGACY) {
2471       // Load immediate 1 into swap_reg %rax
2472       __ movl(swap_reg, 1);
2473 
2474       // Load (object->mark() | 1) into swap_reg %rax
2475       __ orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2476       if (EnableValhalla) {
2477         // Mask inline_type bit such that we go to the slow path if object is an inline type
2478         __ andptr(swap_reg, ~((int) markWord::inline_type_bit_in_place));
2479       }
2480 
2481       // Save (object->mark() | 1) into BasicLock's displaced header
2482       __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2483 
2484       // src -> dest iff dest == rax else rax <- dest
2485       __ lock();
2486       __ cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2487       __ jcc(Assembler::equal, count_mon);
2488 
2489       // Hmm should this move to the slow path code area???
2490 
2491       // Test if the oopMark is an obvious stack pointer, i.e.,
2492       //  1) (mark & 3) == 0, and
2493       //  2) rsp <= mark < mark + os::pagesize()
2494       // These 3 tests can be done by evaluating the following
2495       // expression: ((mark - rsp) & (3 - os::vm_page_size())),
2496       // assuming both stack pointer and pagesize have their
2497       // least significant 2 bits clear.
2498       // NOTE: the oopMark is in swap_reg %rax as the result of cmpxchg
2499 

3887   julong *scratch = (julong *)alloca(total_allocation);
3888 
3889   // Local scratch arrays
3890   julong
3891     *a = scratch + 0 * longwords,
3892     *n = scratch + 1 * longwords,
3893     *m = scratch + 2 * longwords;
3894 
3895   reverse_words((julong *)a_ints, a, longwords);
3896   reverse_words((julong *)n_ints, n, longwords);
3897 
3898   if (len >= MONTGOMERY_SQUARING_THRESHOLD) {
3899     ::montgomery_square(a, n, m, (julong)inv, longwords);
3900   } else {
3901     ::montgomery_multiply(a, a, n, m, (julong)inv, longwords);
3902   }
3903 
3904   reverse_words(m, (julong *)m_ints, longwords);
3905 }
3906 
3907 BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(const InlineKlass* vk) {
3908   BufferBlob* buf = BufferBlob::create("inline types pack/unpack", 16 * K);
3909   CodeBuffer buffer(buf);
3910   short buffer_locs[20];
3911   buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
3912                                          sizeof(buffer_locs)/sizeof(relocInfo));
3913 
3914   MacroAssembler* masm = new MacroAssembler(&buffer);
3915 
3916   const Array<SigEntry>* sig_vk = vk->extended_sig();
3917   const Array<VMRegPair>* regs = vk->return_regs();
3918 
3919   int pack_fields_jobject_off = __ offset();
3920   // Resolve pre-allocated buffer from JNI handle.
3921   // We cannot do this in generate_call_stub() because it requires GC code to be initialized.
3922   __ movptr(rax, Address(r13, 0));
3923   __ resolve_jobject(rax /* value */,
3924                      r12 /* tmp */);
3925   __ movptr(Address(r13, 0), rax);
3926 
3927   int pack_fields_off = __ offset();
3928 
3929   int j = 1;
3930   for (int i = 0; i < sig_vk->length(); i++) {
3931     BasicType bt = sig_vk->at(i)._bt;
3932     if (bt == T_METADATA) {
3933       continue;
3934     }
3935     if (bt == T_VOID) {
3936       if (sig_vk->at(i-1)._bt == T_LONG ||
3937           sig_vk->at(i-1)._bt == T_DOUBLE) {
3938         j++;
3939       }
3940       continue;
3941     }
3942     int off = sig_vk->at(i)._offset;
3943     assert(off > 0, "offset in object should be positive");
3944     VMRegPair pair = regs->at(j);
3945     VMReg r_1 = pair.first();
3946     VMReg r_2 = pair.second();
3947     Address to(rax, off);
3948     if (bt == T_FLOAT) {
3949       __ movflt(to, r_1->as_XMMRegister());
3950     } else if (bt == T_DOUBLE) {
3951       __ movdbl(to, r_1->as_XMMRegister());
3952     } else {
3953       Register val = r_1->as_Register();
3954       assert_different_registers(to.base(), val, r14, r13, rbx, rscratch1);
3955       if (is_reference_type(bt)) {
3956         __ store_heap_oop(to, val, r14, r13, rbx, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
3957       } else {
3958         __ store_sized_value(to, r_1->as_Register(), type2aelembytes(bt));
3959       }
3960     }
3961     j++;
3962   }
3963   assert(j == regs->length(), "missed a field?");
3964   if (vk->has_nullable_atomic_layout()) {
3965     // Set the null marker
3966     __ movb(Address(rax, vk->null_marker_offset()), 1);
3967   }
3968   __ ret(0);
3969 
3970   int unpack_fields_off = __ offset();
3971 
3972   Label skip;
3973   Label not_null;
3974   __ testptr(rax, rax);
3975   __ jcc(Assembler::notZero, not_null);
3976 
3977   // Return value is null. Zero oop registers to make the GC happy.
3978   j = 1;
3979   for (int i = 0; i < sig_vk->length(); i++) {
3980     BasicType bt = sig_vk->at(i)._bt;
3981     if (bt == T_METADATA) {
3982       continue;
3983     }
3984     if (bt == T_VOID) {
3985       if (sig_vk->at(i-1)._bt == T_LONG ||
3986           sig_vk->at(i-1)._bt == T_DOUBLE) {
3987         j++;
3988       }
3989       continue;
3990     }
3991     if (bt == T_OBJECT || bt == T_ARRAY) {
3992       VMRegPair pair = regs->at(j);
3993       VMReg r_1 = pair.first();
3994       __ xorq(r_1->as_Register(), r_1->as_Register());
3995     }
3996     j++;
3997   }
3998   __ jmp(skip);
3999   __ bind(not_null);
4000 
4001   j = 1;
4002   for (int i = 0; i < sig_vk->length(); i++) {
4003     BasicType bt = sig_vk->at(i)._bt;
4004     if (bt == T_METADATA) {
4005       continue;
4006     }
4007     if (bt == T_VOID) {
4008       if (sig_vk->at(i-1)._bt == T_LONG ||
4009           sig_vk->at(i-1)._bt == T_DOUBLE) {
4010         j++;
4011       }
4012       continue;
4013     }
4014     int off = sig_vk->at(i)._offset;
4015     assert(off > 0, "offset in object should be positive");
4016     VMRegPair pair = regs->at(j);
4017     VMReg r_1 = pair.first();
4018     VMReg r_2 = pair.second();
4019     Address from(rax, off);
4020     if (bt == T_FLOAT) {
4021       __ movflt(r_1->as_XMMRegister(), from);
4022     } else if (bt == T_DOUBLE) {
4023       __ movdbl(r_1->as_XMMRegister(), from);
4024     } else if (bt == T_OBJECT || bt == T_ARRAY) {
4025       assert_different_registers(rax, r_1->as_Register());
4026       __ load_heap_oop(r_1->as_Register(), from);
4027     } else {
4028       assert(is_java_primitive(bt), "unexpected basic type");
4029       assert_different_registers(rax, r_1->as_Register());
4030       size_t size_in_bytes = type2aelembytes(bt);
4031       __ load_sized_value(r_1->as_Register(), from, size_in_bytes, bt != T_CHAR && bt != T_BOOLEAN);
4032     }
4033     j++;
4034   }
4035   assert(j == regs->length(), "missed a field?");
4036 
4037   __ bind(skip);
4038   __ ret(0);
4039 
4040   __ flush();
4041 
4042   return BufferedInlineTypeBlob::create(&buffer, pack_fields_off, pack_fields_jobject_off, unpack_fields_off);
4043 }
4044 
4045 #if INCLUDE_JFR
4046 
4047 // For c2: c_rarg0 is junk, call to runtime to write a checkpoint.
4048 // It returns a jobject handle to the event writer.
4049 // The handle is dereferenced and the return value is the event writer oop.
4050 RuntimeStub* SharedRuntime::generate_jfr_write_checkpoint() {
4051   enum layout {
4052     rbp_off,
4053     rbpH_off,
4054     return_off,
4055     return_off2,
4056     framesize // inclusive of return address
4057   };
4058 
4059   const char* name = SharedRuntime::stub_name(SharedStubId::jfr_write_checkpoint_id);
4060   CodeBuffer code(name, 1024, 64);
4061   MacroAssembler* masm = new MacroAssembler(&code);
4062   address start = __ pc();
4063 
4064   __ enter();

4117   __ reset_last_Java_frame(true);
4118 
4119   __ leave();
4120   __ ret(0);
4121 
4122   OopMapSet* oop_maps = new OopMapSet();
4123   OopMap* map = new OopMap(framesize, 1);
4124   oop_maps->add_gc_map(frame_complete, map);
4125 
4126   RuntimeStub* stub =
4127     RuntimeStub::new_runtime_stub(name,
4128                                   &code,
4129                                   frame_complete,
4130                                   (framesize >> (LogBytesPerWord - LogBytesPerInt)),
4131                                   oop_maps,
4132                                   false);
4133   return stub;
4134 }
4135 
4136 #endif // INCLUDE_JFR

< prev index next >