< prev index next >

src/hotspot/cpu/x86/stubGenerator_x86_64.cpp

Print this page

   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 

  25 #include "asm/macroAssembler.hpp"
  26 #include "classfile/javaClasses.hpp"
  27 #include "classfile/vmIntrinsics.hpp"
  28 #include "compiler/oopMap.hpp"
  29 #include "gc/shared/barrierSet.hpp"
  30 #include "gc/shared/barrierSetAssembler.hpp"
  31 #include "gc/shared/barrierSetNMethod.hpp"
  32 #include "gc/shared/gc_globals.hpp"
  33 #include "memory/universe.hpp"
  34 #include "prims/jvmtiExport.hpp"
  35 #include "prims/upcallLinker.hpp"
  36 #include "runtime/arguments.hpp"
  37 #include "runtime/continuationEntry.hpp"
  38 #include "runtime/javaThread.hpp"
  39 #include "runtime/sharedRuntime.hpp"
  40 #include "runtime/stubRoutines.hpp"


  41 #include "stubGenerator_x86_64.hpp"
  42 #ifdef COMPILER2
  43 #include "opto/runtime.hpp"
  44 #include "opto/c2_globals.hpp"
  45 #endif
  46 #if INCLUDE_JVMCI
  47 #include "jvmci/jvmci_globals.hpp"
  48 #endif
  49 
  50 // For a more detailed description of the stub routine structure
  51 // see the comment in stubRoutines.hpp
  52 
  53 #define __ _masm->
  54 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
  55 
  56 #ifdef PRODUCT
  57 #define BLOCK_COMMENT(str) /* nothing */
  58 #else
  59 #define BLOCK_COMMENT(str) __ block_comment(str)
  60 #endif // PRODUCT

 284   __ BIND(loop);
 285   __ movptr(rax, Address(c_rarg2, 0));// get parameter
 286   __ addptr(c_rarg2, wordSize);       // advance to next parameter
 287   __ decrementl(c_rarg1);             // decrement counter
 288   __ push(rax);                       // pass parameter
 289   __ jcc(Assembler::notZero, loop);
 290 
 291   // call Java function
 292   __ BIND(parameters_done);
 293   __ movptr(rbx, method);             // get Method*
 294   __ movptr(c_rarg1, entry_point);    // get entry_point
 295   __ mov(r13, rsp);                   // set sender sp
 296   BLOCK_COMMENT("call Java function");
 297   __ call(c_rarg1);
 298 
 299   BLOCK_COMMENT("call_stub_return_address:");
 300   return_address = __ pc();
 301 
 302   // store result depending on type (everything that is not
 303   // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 304   __ movptr(c_rarg0, result);
 305   Label is_long, is_float, is_double, exit;
 306   __ movl(c_rarg1, result_type);
 307   __ cmpl(c_rarg1, T_OBJECT);


 308   __ jcc(Assembler::equal, is_long);
 309   __ cmpl(c_rarg1, T_LONG);
 310   __ jcc(Assembler::equal, is_long);
 311   __ cmpl(c_rarg1, T_FLOAT);
 312   __ jcc(Assembler::equal, is_float);
 313   __ cmpl(c_rarg1, T_DOUBLE);
 314   __ jcc(Assembler::equal, is_double);
 315 #ifdef ASSERT
 316   // make sure the type is INT
 317   {
 318     Label L;
 319     __ cmpl(c_rarg1, T_INT);
 320     __ jcc(Assembler::equal, L);
 321     __ stop("StubRoutines::call_stub: unexpected result type");
 322     __ bind(L);
 323   }
 324 #endif
 325 
 326   // handle T_INT case
 327   __ movl(Address(c_rarg0, 0), rax);
 328 
 329   __ BIND(exit);
 330 
 331   // pop parameters
 332   __ lea(rsp, rsp_after_call);
 333 
 334 #ifdef ASSERT
 335   // verify that threads correspond
 336   {
 337    Label L1, L2, L3;
 338     __ cmpptr(r15_thread, thread);
 339     __ jcc(Assembler::equal, L1);
 340     __ stop("StubRoutines::call_stub: r15_thread is corrupted");
 341     __ bind(L1);
 342     __ get_thread_slow(rbx);
 343     __ cmpptr(r15_thread, thread);
 344     __ jcc(Assembler::equal, L2);
 345     __ stop("StubRoutines::call_stub: r15_thread is modified by call");
 346     __ bind(L2);
 347     __ cmpptr(r15_thread, rbx);

 365   __ movptr(r13, r13_save);
 366   __ movptr(r12, r12_save);
 367   __ movptr(rbx, rbx_save);
 368 
 369 #ifdef _WIN64
 370   __ movptr(rdi, rdi_save);
 371   __ movptr(rsi, rsi_save);
 372 #else
 373   __ ldmxcsr(mxcsr_save);
 374 #endif
 375 
 376   // restore rsp
 377   __ addptr(rsp, -rsp_after_call_off * wordSize);
 378 
 379   // return
 380   __ vzeroupper();
 381   __ pop(rbp);
 382   __ ret(0);
 383 
 384   // handle return types different from T_INT













 385   __ BIND(is_long);
 386   __ movq(Address(c_rarg0, 0), rax);
 387   __ jmp(exit);
 388 
 389   __ BIND(is_float);
 390   __ movflt(Address(c_rarg0, 0), xmm0);
 391   __ jmp(exit);
 392 
 393   __ BIND(is_double);
 394   __ movdbl(Address(c_rarg0, 0), xmm0);
 395   __ jmp(exit);
 396 
 397   return start;
 398 }
 399 
 400 // Return point for a Java call if there's an exception thrown in
 401 // Java code.  The exception is caught and transformed into a
 402 // pending exception stored in JavaThread that can be tested from
 403 // within the VM.
 404 //
 405 // Note: Usually the parameters are removed by the callee. In case
 406 // of an exception crossing an activation frame boundary, that is
 407 // not the case if the callee is compiled code => need to setup the
 408 // rsp.
 409 //
 410 // rax: exception oop
 411 
 412 address StubGenerator::generate_catch_exception() {
 413   StubId stub_id = StubId::stubgen_catch_exception_id;
 414   StubCodeMark mark(this, stub_id);

3748 *  Output:
3749 *        rax   - float16  jshort
3750 */
3751 address StubGenerator::generate_floatToFloat16() {
3752   StubId stub_id = StubId::stubgen_f2hf_id;
3753   StubCodeMark mark(this, stub_id);
3754 
3755   address start = __ pc();
3756 
3757   BLOCK_COMMENT("Entry:");
3758   // No need for RuntimeStub frame since it is called only during JIT compilation
3759 
3760   // Convert and put result into rax
3761   __ flt_to_flt16(rax, xmm0, xmm1);
3762 
3763   __ ret(0);
3764 
3765   return start;
3766 }
3767 





























































3768 address StubGenerator::generate_cont_thaw(StubId stub_id) {
3769   if (!Continuations::enabled()) return nullptr;
3770 
3771   bool return_barrier;
3772   bool return_barrier_exception;
3773   Continuation::thaw_kind kind;
3774 
3775   switch (stub_id) {
3776   case StubId::stubgen_cont_thaw_id:
3777     return_barrier = false;
3778     return_barrier_exception = false;
3779     kind = Continuation::thaw_top;
3780     break;
3781   case StubId::stubgen_cont_returnBarrier_id:
3782     return_barrier = true;
3783     return_barrier_exception = false;
3784     kind = Continuation::thaw_return_barrier;
3785     break;
3786   case StubId::stubgen_cont_returnBarrierExc_id:
3787     return_barrier = true;

3799   if (!return_barrier) {
3800     // Pop return address. If we don't do this, we get a drift,
3801     // where the bottom-most frozen frame continuously grows.
3802     __ pop(c_rarg3);
3803   } else {
3804     __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
3805   }
3806 
3807 #ifdef ASSERT
3808   {
3809     Label L_good_sp;
3810     __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
3811     __ jcc(Assembler::equal, L_good_sp);
3812     __ stop("Incorrect rsp at thaw entry");
3813     __ BIND(L_good_sp);
3814   }
3815 #endif // ASSERT
3816 
3817   if (return_barrier) {
3818     // Preserve possible return value from a method returning to the return barrier.
3819     __ push_ppx(rax);
3820     __ push_d(xmm0);
3821   }
3822 
3823   __ movptr(c_rarg0, r15_thread);
3824   __ movptr(c_rarg1, (return_barrier ? 1 : 0));
3825   __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), 2);
3826   __ movptr(rbx, rax);
3827 
3828   if (return_barrier) {
3829     // Restore return value from a method returning to the return barrier.
3830     // No safepoint in the call to thaw, so even an oop return value should be OK.
3831     __ pop_d(xmm0);
3832     __ pop_ppx(rax);
3833   }
3834 
3835 #ifdef ASSERT
3836   {
3837     Label L_good_sp;
3838     __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
3839     __ jcc(Assembler::equal, L_good_sp);
3840     __ stop("Incorrect rsp after prepare thaw");
3841     __ BIND(L_good_sp);
3842   }
3843 #endif // ASSERT
3844 
3845   // rbx contains the size of the frames to thaw, 0 if overflow or no more frames
3846   Label L_thaw_success;
3847   __ testptr(rbx, rbx);
3848   __ jccb(Assembler::notZero, L_thaw_success);
3849   __ jump(RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
3850   __ bind(L_thaw_success);
3851 
3852   // Make room for the thawed frames and align the stack.
3853   __ subptr(rsp, rbx);
3854   __ andptr(rsp, -StackAlignmentInBytes);
3855 
3856   if (return_barrier) {
3857     // Preserve possible return value from a method returning to the return barrier. (Again.)
3858     __ push_ppx(rax);
3859     __ push_d(xmm0);
3860   }
3861 
3862   // If we want, we can templatize thaw by kind, and have three different entries.
3863   __ movptr(c_rarg0, r15_thread);
3864   __ movptr(c_rarg1, kind);
3865   __ call_VM_leaf(Continuation::thaw_entry(), 2);
3866   __ movptr(rbx, rax);
3867 
3868   if (return_barrier) {
3869     // Restore return value from a method returning to the return barrier. (Again.)
3870     // No safepoint in the call to thaw, so even an oop return value should be OK.
3871     __ pop_d(xmm0);
3872     __ pop_ppx(rax);
3873   } else {
3874     // Return 0 (success) from doYield.
3875     __ xorptr(rax, rax);
3876   }
3877 
3878   // After thawing, rbx is the SP of the yielding frame.
3879   // Move there, and then to saved RBP slot.
3880   __ movptr(rsp, rbx);
3881   __ subptr(rsp, 2*wordSize);
3882 
3883   if (return_barrier_exception) {
3884     __ movptr(c_rarg0, r15_thread);
3885     __ movptr(c_rarg1, Address(rsp, wordSize)); // return address
3886 
3887     // rax still holds the original exception oop, save it before the call
3888     __ push_ppx(rax);
3889 
3890     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), 2);
3891     __ movptr(rbx, rax);
3892 

4059 
4060 void StubGenerator::generate_initial_stubs() {
4061   // Generates all stubs and initializes the entry points
4062 
4063   // This platform-specific settings are needed by generate_call_stub()
4064   create_control_words();
4065 
4066   // Initialize table for unsafe copy memeory check.
4067   if (UnsafeMemoryAccess::_table == nullptr) {
4068     UnsafeMemoryAccess::create_table(16 + 4); // 16 for copyMemory; 4 for setMemory
4069   }
4070 
4071   // entry points that exist in all platforms Note: This is code
4072   // that could be shared among different platforms - however the
4073   // benefit seems to be smaller than the disadvantage of having a
4074   // much more complicated generator structure. See also comment in
4075   // stubRoutines.hpp.
4076 
4077   StubRoutines::_forward_exception_entry = generate_forward_exception();
4078 










4079   StubRoutines::_call_stub_entry =
4080     generate_call_stub(StubRoutines::_call_stub_return_address);
4081 
4082   // is referenced by megamorphic call
4083   StubRoutines::_catch_exception_entry = generate_catch_exception();
4084 
4085   // platform dependent
4086   StubRoutines::x86::_get_previous_sp_entry = generate_get_previous_sp();
4087 
4088   StubRoutines::x86::_verify_mxcsr_entry    = generate_verify_mxcsr();
4089 
4090   StubRoutines::x86::_f2i_fixup             = generate_f2i_fixup();
4091   StubRoutines::x86::_f2l_fixup             = generate_f2l_fixup();
4092   StubRoutines::x86::_d2i_fixup             = generate_d2i_fixup();
4093   StubRoutines::x86::_d2l_fixup             = generate_d2l_fixup();
4094 
4095   StubRoutines::x86::_float_sign_mask       = generate_fp_mask(StubId::stubgen_float_sign_mask_id,  0x7FFFFFFF7FFFFFFF);
4096   StubRoutines::x86::_float_sign_flip       = generate_fp_mask(StubId::stubgen_float_sign_flip_id,  0x8000000080000000);
4097   StubRoutines::x86::_double_sign_mask      = generate_fp_mask(StubId::stubgen_double_sign_mask_id, 0x7FFFFFFFFFFFFFFF);
4098   StubRoutines::x86::_double_sign_flip      = generate_fp_mask(StubId::stubgen_double_sign_flip_id, 0x8000000000000000);

4104   if (UseCRC32CIntrinsics) {
4105     bool supports_clmul = VM_Version::supports_clmul();
4106     StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
4107   }
4108 
4109   if (VM_Version::supports_float16()) {
4110     // For results consistency both intrinsics should be enabled.
4111     // vmIntrinsics checks InlineIntrinsics flag, no need to check it here.
4112     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
4113         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
4114       StubRoutines::_hf2f = generate_float16ToFloat();
4115       StubRoutines::_f2hf = generate_floatToFloat16();
4116     }
4117   }
4118 
4119   generate_libm_stubs();
4120 
4121   StubRoutines::_fmod = generate_libmFmod(); // from stubGenerator_x86_64_fmod.cpp
4122 }
4123 
















































































































































4124 void StubGenerator::generate_continuation_stubs() {
4125   // Continuation stubs:
4126   StubRoutines::_cont_thaw          = generate_cont_thaw();
4127   StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
4128   StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
4129   StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
4130 }
4131 
4132 void StubGenerator::generate_final_stubs() {
4133   // Generates the rest of stubs and initializes the entry points
4134 
4135   // support for verify_oop (must happen after universe_init)
4136   if (VerifyOops) {
4137     StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
4138   }
4139 
4140   // arraycopy stubs used by compilers
4141   generate_arraycopy_stubs();
4142 
4143   StubRoutines::_method_entry_barrier = generate_method_entry_barrier();

   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/assembler.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "classfile/javaClasses.hpp"
  28 #include "classfile/vmIntrinsics.hpp"
  29 #include "compiler/oopMap.hpp"
  30 #include "gc/shared/barrierSet.hpp"
  31 #include "gc/shared/barrierSetAssembler.hpp"
  32 #include "gc/shared/barrierSetNMethod.hpp"
  33 #include "gc/shared/gc_globals.hpp"
  34 #include "memory/universe.hpp"
  35 #include "prims/jvmtiExport.hpp"
  36 #include "prims/upcallLinker.hpp"
  37 #include "runtime/arguments.hpp"
  38 #include "runtime/continuationEntry.hpp"
  39 #include "runtime/javaThread.hpp"
  40 #include "runtime/sharedRuntime.hpp"
  41 #include "runtime/stubRoutines.hpp"
  42 #include "utilities/macros.hpp"
  43 #include "vmreg_x86.inline.hpp"
  44 #include "stubGenerator_x86_64.hpp"
  45 #ifdef COMPILER2
  46 #include "opto/runtime.hpp"
  47 #include "opto/c2_globals.hpp"
  48 #endif
  49 #if INCLUDE_JVMCI
  50 #include "jvmci/jvmci_globals.hpp"
  51 #endif
  52 
  53 // For a more detailed description of the stub routine structure
  54 // see the comment in stubRoutines.hpp
  55 
  56 #define __ _masm->
  57 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
  58 
  59 #ifdef PRODUCT
  60 #define BLOCK_COMMENT(str) /* nothing */
  61 #else
  62 #define BLOCK_COMMENT(str) __ block_comment(str)
  63 #endif // PRODUCT

 287   __ BIND(loop);
 288   __ movptr(rax, Address(c_rarg2, 0));// get parameter
 289   __ addptr(c_rarg2, wordSize);       // advance to next parameter
 290   __ decrementl(c_rarg1);             // decrement counter
 291   __ push(rax);                       // pass parameter
 292   __ jcc(Assembler::notZero, loop);
 293 
 294   // call Java function
 295   __ BIND(parameters_done);
 296   __ movptr(rbx, method);             // get Method*
 297   __ movptr(c_rarg1, entry_point);    // get entry_point
 298   __ mov(r13, rsp);                   // set sender sp
 299   BLOCK_COMMENT("call Java function");
 300   __ call(c_rarg1);
 301 
 302   BLOCK_COMMENT("call_stub_return_address:");
 303   return_address = __ pc();
 304 
 305   // store result depending on type (everything that is not
 306   // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 307   __ movptr(r13, result);
 308   Label is_long, is_float, is_double, check_prim, exit;
 309   __ movl(rbx, result_type);
 310   __ cmpl(rbx, T_OBJECT);
 311   __ jcc(Assembler::equal, check_prim);
 312   __ cmpl(rbx, T_LONG);
 313   __ jcc(Assembler::equal, is_long);
 314   __ cmpl(rbx, T_FLOAT);


 315   __ jcc(Assembler::equal, is_float);
 316   __ cmpl(rbx, T_DOUBLE);
 317   __ jcc(Assembler::equal, is_double);
 318 #ifdef ASSERT
 319   // make sure the type is INT
 320   {
 321     Label L;
 322     __ cmpl(rbx, T_INT);
 323     __ jcc(Assembler::equal, L);
 324     __ stop("StubRoutines::call_stub: unexpected result type");
 325     __ bind(L);
 326   }
 327 #endif
 328 
 329   // handle T_INT case
 330   __ movl(Address(r13, 0), rax);
 331 
 332   __ BIND(exit);
 333 
 334   // pop parameters
 335   __ lea(rsp, rsp_after_call);
 336 
 337 #ifdef ASSERT
 338   // verify that threads correspond
 339   {
 340    Label L1, L2, L3;
 341     __ cmpptr(r15_thread, thread);
 342     __ jcc(Assembler::equal, L1);
 343     __ stop("StubRoutines::call_stub: r15_thread is corrupted");
 344     __ bind(L1);
 345     __ get_thread_slow(rbx);
 346     __ cmpptr(r15_thread, thread);
 347     __ jcc(Assembler::equal, L2);
 348     __ stop("StubRoutines::call_stub: r15_thread is modified by call");
 349     __ bind(L2);
 350     __ cmpptr(r15_thread, rbx);

 368   __ movptr(r13, r13_save);
 369   __ movptr(r12, r12_save);
 370   __ movptr(rbx, rbx_save);
 371 
 372 #ifdef _WIN64
 373   __ movptr(rdi, rdi_save);
 374   __ movptr(rsi, rsi_save);
 375 #else
 376   __ ldmxcsr(mxcsr_save);
 377 #endif
 378 
 379   // restore rsp
 380   __ addptr(rsp, -rsp_after_call_off * wordSize);
 381 
 382   // return
 383   __ vzeroupper();
 384   __ pop(rbp);
 385   __ ret(0);
 386 
 387   // handle return types different from T_INT
 388   __ BIND(check_prim);
 389   if (InlineTypeReturnedAsFields) {
 390     // Check for scalarized return value
 391     __ testptr(rax, 1);
 392     __ jcc(Assembler::zero, is_long);
 393     // Load pack handler address
 394     __ andptr(rax, -2);
 395     __ movptr(rax, Address(rax, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 396     __ movptr(rbx, Address(rax, InlineKlass::pack_handler_jobject_offset()));
 397     // Call pack handler to initialize the buffer
 398     __ call(rbx);
 399     __ jmp(exit);
 400   }
 401   __ BIND(is_long);
 402   __ movq(Address(r13, 0), rax);
 403   __ jmp(exit);
 404 
 405   __ BIND(is_float);
 406   __ movflt(Address(r13, 0), xmm0);
 407   __ jmp(exit);
 408 
 409   __ BIND(is_double);
 410   __ movdbl(Address(r13, 0), xmm0);
 411   __ jmp(exit);
 412 
 413   return start;
 414 }
 415 
 416 // Return point for a Java call if there's an exception thrown in
 417 // Java code.  The exception is caught and transformed into a
 418 // pending exception stored in JavaThread that can be tested from
 419 // within the VM.
 420 //
 421 // Note: Usually the parameters are removed by the callee. In case
 422 // of an exception crossing an activation frame boundary, that is
 423 // not the case if the callee is compiled code => need to setup the
 424 // rsp.
 425 //
 426 // rax: exception oop
 427 
 428 address StubGenerator::generate_catch_exception() {
 429   StubId stub_id = StubId::stubgen_catch_exception_id;
 430   StubCodeMark mark(this, stub_id);

3764 *  Output:
3765 *        rax   - float16  jshort
3766 */
3767 address StubGenerator::generate_floatToFloat16() {
3768   StubId stub_id = StubId::stubgen_f2hf_id;
3769   StubCodeMark mark(this, stub_id);
3770 
3771   address start = __ pc();
3772 
3773   BLOCK_COMMENT("Entry:");
3774   // No need for RuntimeStub frame since it is called only during JIT compilation
3775 
3776   // Convert and put result into rax
3777   __ flt_to_flt16(rax, xmm0, xmm1);
3778 
3779   __ ret(0);
3780 
3781   return start;
3782 }
3783 
3784 static void save_return_registers(MacroAssembler* masm) {
3785   masm->push_ppx(rax);
3786   if (InlineTypeReturnedAsFields) {
3787     masm->push(rdi);
3788     masm->push(rsi);
3789     masm->push(rdx);
3790     masm->push(rcx);
3791     masm->push(r8);
3792     masm->push(r9);
3793   }
3794   masm->push_d(xmm0);
3795   if (InlineTypeReturnedAsFields) {
3796     masm->push_d(xmm1);
3797     masm->push_d(xmm2);
3798     masm->push_d(xmm3);
3799     masm->push_d(xmm4);
3800     masm->push_d(xmm5);
3801     masm->push_d(xmm6);
3802     masm->push_d(xmm7);
3803   }
3804 #ifdef ASSERT
3805   masm->movq(rax, 0xBADC0FFE);
3806   masm->movq(rdi, rax);
3807   masm->movq(rsi, rax);
3808   masm->movq(rdx, rax);
3809   masm->movq(rcx, rax);
3810   masm->movq(r8, rax);
3811   masm->movq(r9, rax);
3812   masm->movq(xmm0, rax);
3813   masm->movq(xmm1, rax);
3814   masm->movq(xmm2, rax);
3815   masm->movq(xmm3, rax);
3816   masm->movq(xmm4, rax);
3817   masm->movq(xmm5, rax);
3818   masm->movq(xmm6, rax);
3819   masm->movq(xmm7, rax);
3820 #endif
3821 }
3822 
3823 static void restore_return_registers(MacroAssembler* masm) {
3824   if (InlineTypeReturnedAsFields) {
3825     masm->pop_d(xmm7);
3826     masm->pop_d(xmm6);
3827     masm->pop_d(xmm5);
3828     masm->pop_d(xmm4);
3829     masm->pop_d(xmm3);
3830     masm->pop_d(xmm2);
3831     masm->pop_d(xmm1);
3832   }
3833   masm->pop_d(xmm0);
3834   if (InlineTypeReturnedAsFields) {
3835     masm->pop(r9);
3836     masm->pop(r8);
3837     masm->pop(rcx);
3838     masm->pop(rdx);
3839     masm->pop(rsi);
3840     masm->pop(rdi);
3841   }
3842   masm->pop_ppx(rax);
3843 }
3844 
3845 address StubGenerator::generate_cont_thaw(StubId stub_id) {
3846   if (!Continuations::enabled()) return nullptr;
3847 
3848   bool return_barrier;
3849   bool return_barrier_exception;
3850   Continuation::thaw_kind kind;
3851 
3852   switch (stub_id) {
3853   case StubId::stubgen_cont_thaw_id:
3854     return_barrier = false;
3855     return_barrier_exception = false;
3856     kind = Continuation::thaw_top;
3857     break;
3858   case StubId::stubgen_cont_returnBarrier_id:
3859     return_barrier = true;
3860     return_barrier_exception = false;
3861     kind = Continuation::thaw_return_barrier;
3862     break;
3863   case StubId::stubgen_cont_returnBarrierExc_id:
3864     return_barrier = true;

3876   if (!return_barrier) {
3877     // Pop return address. If we don't do this, we get a drift,
3878     // where the bottom-most frozen frame continuously grows.
3879     __ pop(c_rarg3);
3880   } else {
3881     __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
3882   }
3883 
3884 #ifdef ASSERT
3885   {
3886     Label L_good_sp;
3887     __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
3888     __ jcc(Assembler::equal, L_good_sp);
3889     __ stop("Incorrect rsp at thaw entry");
3890     __ BIND(L_good_sp);
3891   }
3892 #endif // ASSERT
3893 
3894   if (return_barrier) {
3895     // Preserve possible return value from a method returning to the return barrier.
3896     save_return_registers(_masm);

3897   }
3898 
3899   __ movptr(c_rarg0, r15_thread);
3900   __ movptr(c_rarg1, (return_barrier ? 1 : 0));
3901   __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), 2);
3902   __ movptr(rbx, rax);
3903 
3904   if (return_barrier) {
3905     // Restore return value from a method returning to the return barrier.
3906     // No safepoint in the call to thaw, so even an oop return value should be OK.
3907     restore_return_registers(_masm);

3908   }
3909 
3910 #ifdef ASSERT
3911   {
3912     Label L_good_sp;
3913     __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
3914     __ jcc(Assembler::equal, L_good_sp);
3915     __ stop("Incorrect rsp after prepare thaw");
3916     __ BIND(L_good_sp);
3917   }
3918 #endif // ASSERT
3919 
3920   // rbx contains the size of the frames to thaw, 0 if overflow or no more frames
3921   Label L_thaw_success;
3922   __ testptr(rbx, rbx);
3923   __ jccb(Assembler::notZero, L_thaw_success);
3924   __ jump(RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
3925   __ bind(L_thaw_success);
3926 
3927   // Make room for the thawed frames and align the stack.
3928   __ subptr(rsp, rbx);
3929   __ andptr(rsp, -StackAlignmentInBytes);
3930 
3931   if (return_barrier) {
3932     // Preserve possible return value from a method returning to the return barrier. (Again.)
3933     save_return_registers(_masm);

3934   }
3935 
3936   // If we want, we can templatize thaw by kind, and have three different entries.
3937   __ movptr(c_rarg0, r15_thread);
3938   __ movptr(c_rarg1, kind);
3939   __ call_VM_leaf(Continuation::thaw_entry(), 2);
3940   __ movptr(rbx, rax);
3941 
3942   if (return_barrier) {
3943     // Restore return value from a method returning to the return barrier. (Again.)
3944     // No safepoint in the call to thaw, so even an oop return value should be OK.
3945     restore_return_registers(_masm);

3946   } else {
3947     // Return 0 (success) from doYield.
3948     __ xorptr(rax, rax);
3949   }
3950 
3951   // After thawing, rbx is the SP of the yielding frame.
3952   // Move there, and then to saved RBP slot.
3953   __ movptr(rsp, rbx);
3954   __ subptr(rsp, 2*wordSize);
3955 
3956   if (return_barrier_exception) {
3957     __ movptr(c_rarg0, r15_thread);
3958     __ movptr(c_rarg1, Address(rsp, wordSize)); // return address
3959 
3960     // rax still holds the original exception oop, save it before the call
3961     __ push_ppx(rax);
3962 
3963     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), 2);
3964     __ movptr(rbx, rax);
3965 

4132 
4133 void StubGenerator::generate_initial_stubs() {
4134   // Generates all stubs and initializes the entry points
4135 
4136   // This platform-specific settings are needed by generate_call_stub()
4137   create_control_words();
4138 
4139   // Initialize table for unsafe copy memeory check.
4140   if (UnsafeMemoryAccess::_table == nullptr) {
4141     UnsafeMemoryAccess::create_table(16 + 4); // 16 for copyMemory; 4 for setMemory
4142   }
4143 
4144   // entry points that exist in all platforms Note: This is code
4145   // that could be shared among different platforms - however the
4146   // benefit seems to be smaller than the disadvantage of having a
4147   // much more complicated generator structure. See also comment in
4148   // stubRoutines.hpp.
4149 
4150   StubRoutines::_forward_exception_entry = generate_forward_exception();
4151 
4152   // Generate these first because they are called from other stubs
4153   if (InlineTypeReturnedAsFields) {
4154     StubRoutines::_load_inline_type_fields_in_regs =
4155       generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::load_inline_type_fields_in_regs),
4156                                  "load_inline_type_fields_in_regs", false);
4157     StubRoutines::_store_inline_type_fields_to_buf =
4158       generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::store_inline_type_fields_to_buf),
4159                                  "store_inline_type_fields_to_buf", true);
4160   }
4161 
4162   StubRoutines::_call_stub_entry =
4163     generate_call_stub(StubRoutines::_call_stub_return_address);
4164 
4165   // is referenced by megamorphic call
4166   StubRoutines::_catch_exception_entry = generate_catch_exception();
4167 
4168   // platform dependent
4169   StubRoutines::x86::_get_previous_sp_entry = generate_get_previous_sp();
4170 
4171   StubRoutines::x86::_verify_mxcsr_entry    = generate_verify_mxcsr();
4172 
4173   StubRoutines::x86::_f2i_fixup             = generate_f2i_fixup();
4174   StubRoutines::x86::_f2l_fixup             = generate_f2l_fixup();
4175   StubRoutines::x86::_d2i_fixup             = generate_d2i_fixup();
4176   StubRoutines::x86::_d2l_fixup             = generate_d2l_fixup();
4177 
4178   StubRoutines::x86::_float_sign_mask       = generate_fp_mask(StubId::stubgen_float_sign_mask_id,  0x7FFFFFFF7FFFFFFF);
4179   StubRoutines::x86::_float_sign_flip       = generate_fp_mask(StubId::stubgen_float_sign_flip_id,  0x8000000080000000);
4180   StubRoutines::x86::_double_sign_mask      = generate_fp_mask(StubId::stubgen_double_sign_mask_id, 0x7FFFFFFFFFFFFFFF);
4181   StubRoutines::x86::_double_sign_flip      = generate_fp_mask(StubId::stubgen_double_sign_flip_id, 0x8000000000000000);

4187   if (UseCRC32CIntrinsics) {
4188     bool supports_clmul = VM_Version::supports_clmul();
4189     StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
4190   }
4191 
4192   if (VM_Version::supports_float16()) {
4193     // For results consistency both intrinsics should be enabled.
4194     // vmIntrinsics checks InlineIntrinsics flag, no need to check it here.
4195     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
4196         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
4197       StubRoutines::_hf2f = generate_float16ToFloat();
4198       StubRoutines::_f2hf = generate_floatToFloat16();
4199     }
4200   }
4201 
4202   generate_libm_stubs();
4203 
4204   StubRoutines::_fmod = generate_libmFmod(); // from stubGenerator_x86_64_fmod.cpp
4205 }
4206 
4207 // Call here from the interpreter or compiled code to either load
4208 // multiple returned values from the inline type instance being
4209 // returned to registers or to store returned values to a newly
4210 // allocated inline type instance.
4211 // Register is a class, but it would be assigned numerical value.
4212 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull.
4213 PRAGMA_DIAG_PUSH
4214 PRAGMA_NONNULL_IGNORED
4215 address StubGenerator::generate_return_value_stub(address destination, const char* name, bool has_res) {
4216   // We need to save all registers the calling convention may use so
4217   // the runtime calls read or update those registers. This needs to
4218   // be in sync with SharedRuntime::java_return_convention().
4219   enum layout {
4220     pad_off = frame::arg_reg_save_area_bytes/BytesPerInt, pad_off_2,
4221     rax_off, rax_off_2,
4222     j_rarg5_off, j_rarg5_2,
4223     j_rarg4_off, j_rarg4_2,
4224     j_rarg3_off, j_rarg3_2,
4225     j_rarg2_off, j_rarg2_2,
4226     j_rarg1_off, j_rarg1_2,
4227     j_rarg0_off, j_rarg0_2,
4228     j_farg0_off, j_farg0_2,
4229     j_farg1_off, j_farg1_2,
4230     j_farg2_off, j_farg2_2,
4231     j_farg3_off, j_farg3_2,
4232     j_farg4_off, j_farg4_2,
4233     j_farg5_off, j_farg5_2,
4234     j_farg6_off, j_farg6_2,
4235     j_farg7_off, j_farg7_2,
4236     rbp_off, rbp_off_2,
4237     return_off, return_off_2,
4238 
4239     framesize
4240   };
4241 
4242   CodeBuffer buffer(name, 1000, 512);
4243   MacroAssembler* _masm = new MacroAssembler(&buffer);
4244 
4245   int frame_size_in_bytes = align_up(framesize*BytesPerInt, 16);
4246   assert(frame_size_in_bytes == framesize*BytesPerInt, "misaligned");
4247   int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
4248   int frame_size_in_words = frame_size_in_bytes / wordSize;
4249 
4250   OopMapSet *oop_maps = new OopMapSet();
4251   OopMap* map = new OopMap(frame_size_in_slots, 0);
4252 
4253   map->set_callee_saved(VMRegImpl::stack2reg(rax_off), rax->as_VMReg());
4254   map->set_callee_saved(VMRegImpl::stack2reg(j_rarg5_off), j_rarg5->as_VMReg());
4255   map->set_callee_saved(VMRegImpl::stack2reg(j_rarg4_off), j_rarg4->as_VMReg());
4256   map->set_callee_saved(VMRegImpl::stack2reg(j_rarg3_off), j_rarg3->as_VMReg());
4257   map->set_callee_saved(VMRegImpl::stack2reg(j_rarg2_off), j_rarg2->as_VMReg());
4258   map->set_callee_saved(VMRegImpl::stack2reg(j_rarg1_off), j_rarg1->as_VMReg());
4259   map->set_callee_saved(VMRegImpl::stack2reg(j_rarg0_off), j_rarg0->as_VMReg());
4260   map->set_callee_saved(VMRegImpl::stack2reg(j_farg0_off), j_farg0->as_VMReg());
4261   map->set_callee_saved(VMRegImpl::stack2reg(j_farg1_off), j_farg1->as_VMReg());
4262   map->set_callee_saved(VMRegImpl::stack2reg(j_farg2_off), j_farg2->as_VMReg());
4263   map->set_callee_saved(VMRegImpl::stack2reg(j_farg3_off), j_farg3->as_VMReg());
4264   map->set_callee_saved(VMRegImpl::stack2reg(j_farg4_off), j_farg4->as_VMReg());
4265   map->set_callee_saved(VMRegImpl::stack2reg(j_farg5_off), j_farg5->as_VMReg());
4266   map->set_callee_saved(VMRegImpl::stack2reg(j_farg6_off), j_farg6->as_VMReg());
4267   map->set_callee_saved(VMRegImpl::stack2reg(j_farg7_off), j_farg7->as_VMReg());
4268 
4269   int start = __ offset();
4270 
4271   __ subptr(rsp, frame_size_in_bytes - 8 /* return address*/);
4272 
4273   __ movptr(Address(rsp, rbp_off * BytesPerInt), rbp);
4274   __ movdbl(Address(rsp, j_farg7_off * BytesPerInt), j_farg7);
4275   __ movdbl(Address(rsp, j_farg6_off * BytesPerInt), j_farg6);
4276   __ movdbl(Address(rsp, j_farg5_off * BytesPerInt), j_farg5);
4277   __ movdbl(Address(rsp, j_farg4_off * BytesPerInt), j_farg4);
4278   __ movdbl(Address(rsp, j_farg3_off * BytesPerInt), j_farg3);
4279   __ movdbl(Address(rsp, j_farg2_off * BytesPerInt), j_farg2);
4280   __ movdbl(Address(rsp, j_farg1_off * BytesPerInt), j_farg1);
4281   __ movdbl(Address(rsp, j_farg0_off * BytesPerInt), j_farg0);
4282 
4283   __ movptr(Address(rsp, j_rarg0_off * BytesPerInt), j_rarg0);
4284   __ movptr(Address(rsp, j_rarg1_off * BytesPerInt), j_rarg1);
4285   __ movptr(Address(rsp, j_rarg2_off * BytesPerInt), j_rarg2);
4286   __ movptr(Address(rsp, j_rarg3_off * BytesPerInt), j_rarg3);
4287   __ movptr(Address(rsp, j_rarg4_off * BytesPerInt), j_rarg4);
4288   __ movptr(Address(rsp, j_rarg5_off * BytesPerInt), j_rarg5);
4289   __ movptr(Address(rsp, rax_off * BytesPerInt), rax);
4290 
4291   int frame_complete = __ offset();
4292 
4293   __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1);
4294 
4295   __ mov(c_rarg0, r15_thread);
4296   __ mov(c_rarg1, rax);
4297 
4298   __ call(RuntimeAddress(destination));
4299 
4300   // Set an oopmap for the call site.
4301 
4302   oop_maps->add_gc_map( __ offset() - start, map);
4303 
4304   // clear last_Java_sp
4305   __ reset_last_Java_frame(false);
4306 
4307   __ movptr(rbp, Address(rsp, rbp_off * BytesPerInt));
4308   __ movdbl(j_farg7, Address(rsp, j_farg7_off * BytesPerInt));
4309   __ movdbl(j_farg6, Address(rsp, j_farg6_off * BytesPerInt));
4310   __ movdbl(j_farg5, Address(rsp, j_farg5_off * BytesPerInt));
4311   __ movdbl(j_farg4, Address(rsp, j_farg4_off * BytesPerInt));
4312   __ movdbl(j_farg3, Address(rsp, j_farg3_off * BytesPerInt));
4313   __ movdbl(j_farg2, Address(rsp, j_farg2_off * BytesPerInt));
4314   __ movdbl(j_farg1, Address(rsp, j_farg1_off * BytesPerInt));
4315   __ movdbl(j_farg0, Address(rsp, j_farg0_off * BytesPerInt));
4316 
4317   __ movptr(j_rarg0, Address(rsp, j_rarg0_off * BytesPerInt));
4318   __ movptr(j_rarg1, Address(rsp, j_rarg1_off * BytesPerInt));
4319   __ movptr(j_rarg2, Address(rsp, j_rarg2_off * BytesPerInt));
4320   __ movptr(j_rarg3, Address(rsp, j_rarg3_off * BytesPerInt));
4321   __ movptr(j_rarg4, Address(rsp, j_rarg4_off * BytesPerInt));
4322   __ movptr(j_rarg5, Address(rsp, j_rarg5_off * BytesPerInt));
4323   __ movptr(rax, Address(rsp, rax_off * BytesPerInt));
4324 
4325   __ addptr(rsp, frame_size_in_bytes-8);
4326 
4327   // check for pending exceptions
4328   Label pending;
4329   __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
4330   __ jcc(Assembler::notEqual, pending);
4331 
4332   if (has_res) {
4333     __ get_vm_result_oop(rax);
4334   }
4335 
4336   __ ret(0);
4337 
4338   __ bind(pending);
4339 
4340   __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
4341   __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
4342 
4343   // -------------
4344   // make sure all code is generated
4345   _masm->flush();
4346 
4347   RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, false);
4348   return stub->entry_point();
4349 }
4350 
4351 void StubGenerator::generate_continuation_stubs() {
4352   // Continuation stubs:
4353   StubRoutines::_cont_thaw          = generate_cont_thaw();
4354   StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
4355   StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
4356   StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
4357 }
4358 
4359 void StubGenerator::generate_final_stubs() {
4360   // Generates the rest of stubs and initializes the entry points
4361 
4362   // support for verify_oop (must happen after universe_init)
4363   if (VerifyOops) {
4364     StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
4365   }
4366 
4367   // arraycopy stubs used by compilers
4368   generate_arraycopy_stubs();
4369 
4370   StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
< prev index next >