5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "classfile/javaClasses.hpp"
27 #include "classfile/vmIntrinsics.hpp"
28 #include "compiler/oopMap.hpp"
29 #include "gc/shared/barrierSet.hpp"
30 #include "gc/shared/barrierSetAssembler.hpp"
31 #include "gc/shared/barrierSetNMethod.hpp"
32 #include "gc/shared/gc_globals.hpp"
33 #include "memory/universe.hpp"
34 #include "prims/jvmtiExport.hpp"
35 #include "prims/upcallLinker.hpp"
36 #include "runtime/arguments.hpp"
37 #include "runtime/continuationEntry.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #include "runtime/stubRoutines.hpp"
41 #include "stubGenerator_x86_64.hpp"
42 #ifdef COMPILER2
43 #include "opto/runtime.hpp"
44 #include "opto/c2_globals.hpp"
45 #endif
46 #if INCLUDE_JVMCI
47 #include "jvmci/jvmci_globals.hpp"
48 #endif
49
50 // For a more detailed description of the stub routine structure
51 // see the comment in stubRoutines.hpp
52
53 #define __ _masm->
54 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
55
56 #ifdef PRODUCT
57 #define BLOCK_COMMENT(str) /* nothing */
58 #else
59 #define BLOCK_COMMENT(str) __ block_comment(str)
60 #endif // PRODUCT
295 __ movptr(rax, Address(c_rarg2, 0));// get parameter
296 __ addptr(c_rarg2, wordSize); // advance to next parameter
297 __ decrementl(c_rarg1); // decrement counter
298 __ push(rax); // pass parameter
299 __ jcc(Assembler::notZero, loop);
300
301 // call Java function
302 __ BIND(parameters_done);
303 __ movptr(rbx, method); // get Method*
304 __ movptr(c_rarg1, entry_point); // get entry_point
305 __ mov(r13, rsp); // set sender sp
306 BLOCK_COMMENT("call Java function");
307 __ call(c_rarg1);
308
309 BLOCK_COMMENT("call_stub_return_address:");
310 return_address = __ pc();
311 entries.append(return_address);
312
313 // store result depending on type (everything that is not
314 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
315 __ movptr(c_rarg0, result);
316 Label is_long, is_float, is_double, exit;
317 __ movl(c_rarg1, result_type);
318 __ cmpl(c_rarg1, T_OBJECT);
319 __ jcc(Assembler::equal, is_long);
320 __ cmpl(c_rarg1, T_LONG);
321 __ jcc(Assembler::equal, is_long);
322 __ cmpl(c_rarg1, T_FLOAT);
323 __ jcc(Assembler::equal, is_float);
324 __ cmpl(c_rarg1, T_DOUBLE);
325 __ jcc(Assembler::equal, is_double);
326 #ifdef ASSERT
327 // make sure the type is INT
328 {
329 Label L;
330 __ cmpl(c_rarg1, T_INT);
331 __ jcc(Assembler::equal, L);
332 __ stop("StubRoutines::call_stub: unexpected result type");
333 __ bind(L);
334 }
335 #endif
336
337 // handle T_INT case
338 __ movl(Address(c_rarg0, 0), rax);
339
340 __ BIND(exit);
341
342 // pop parameters
343 __ lea(rsp, rsp_after_call);
344
345 #ifdef ASSERT
346 // verify that threads correspond
347 {
348 Label L1, L2, L3;
349 __ cmpptr(r15_thread, thread);
350 __ jcc(Assembler::equal, L1);
351 __ stop("StubRoutines::call_stub: r15_thread is corrupted");
352 __ bind(L1);
353 __ get_thread_slow(rbx);
354 __ cmpptr(r15_thread, thread);
355 __ jcc(Assembler::equal, L2);
356 __ stop("StubRoutines::call_stub: r15_thread is modified by call");
357 __ bind(L2);
358 __ cmpptr(r15_thread, rbx);
376 __ movptr(r13, r13_save);
377 __ movptr(r12, r12_save);
378 __ movptr(rbx, rbx_save);
379
380 #ifdef _WIN64
381 __ movptr(rdi, rdi_save);
382 __ movptr(rsi, rsi_save);
383 #else
384 __ ldmxcsr(mxcsr_save);
385 #endif
386
387 // restore rsp
388 __ addptr(rsp, -rsp_after_call_off * wordSize);
389
390 // return
391 __ vzeroupper();
392 __ pop(rbp);
393 __ ret(0);
394
395 // handle return types different from T_INT
396 __ BIND(is_long);
397 __ movq(Address(c_rarg0, 0), rax);
398 __ jmp(exit);
399
400 __ BIND(is_float);
401 __ movflt(Address(c_rarg0, 0), xmm0);
402 __ jmp(exit);
403
404 __ BIND(is_double);
405 __ movdbl(Address(c_rarg0, 0), xmm0);
406 __ jmp(exit);
407
408 // record the stub entry and end plus the auxiliary entry
409 store_archive_data(stub_id, start, __ pc(), &entries);
410
411 return start;
412 }
413
414 // Return point for a Java call if there's an exception thrown in
415 // Java code. The exception is caught and transformed into a
416 // pending exception stored in JavaThread that can be tested from
417 // within the VM.
418 //
419 // Note: Usually the parameters are removed by the callee. In case
420 // of an exception crossing an activation frame boundary, that is
421 // not the case if the callee is compiled code => need to setup the
422 // rsp.
423 //
424 // rax: exception oop
425
4356 return start;
4357 }
4358 StubCodeMark mark(this, stub_id);
4359
4360 start = __ pc();
4361
4362 BLOCK_COMMENT("Entry:");
4363 // No need for RuntimeStub frame since it is called only during JIT compilation
4364
4365 // Convert and put result into rax
4366 __ flt_to_flt16(rax, xmm0, xmm1);
4367
4368 __ ret(0);
4369
4370 // record the stub entry and end
4371 store_archive_data(stub_id, start, __ pc());
4372
4373 return start;
4374 }
4375
4376 address StubGenerator::generate_cont_thaw(StubId stub_id) {
4377 if (!Continuations::enabled()) return nullptr;
4378
4379 bool return_barrier;
4380 bool return_barrier_exception;
4381 Continuation::thaw_kind kind;
4382
4383 switch (stub_id) {
4384 case StubId::stubgen_cont_thaw_id:
4385 return_barrier = false;
4386 return_barrier_exception = false;
4387 kind = Continuation::thaw_top;
4388 break;
4389 case StubId::stubgen_cont_returnBarrier_id:
4390 return_barrier = true;
4391 return_barrier_exception = false;
4392 kind = Continuation::thaw_return_barrier;
4393 break;
4394 case StubId::stubgen_cont_returnBarrierExc_id:
4395 return_barrier = true;
4413 if (!return_barrier) {
4414 // Pop return address. If we don't do this, we get a drift,
4415 // where the bottom-most frozen frame continuously grows.
4416 __ pop(c_rarg3);
4417 } else {
4418 __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
4419 }
4420
4421 #ifdef ASSERT
4422 {
4423 Label L_good_sp;
4424 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
4425 __ jcc(Assembler::equal, L_good_sp);
4426 __ stop("Incorrect rsp at thaw entry");
4427 __ BIND(L_good_sp);
4428 }
4429 #endif // ASSERT
4430
4431 if (return_barrier) {
4432 // Preserve possible return value from a method returning to the return barrier.
4433 __ push_ppx(rax);
4434 __ push_d(xmm0);
4435 }
4436
4437 __ movptr(c_rarg0, r15_thread);
4438 __ movptr(c_rarg1, (return_barrier ? 1 : 0));
4439 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), 2);
4440 __ movptr(rbx, rax);
4441
4442 if (return_barrier) {
4443 // Restore return value from a method returning to the return barrier.
4444 // No safepoint in the call to thaw, so even an oop return value should be OK.
4445 __ pop_d(xmm0);
4446 __ pop_ppx(rax);
4447 }
4448
4449 #ifdef ASSERT
4450 {
4451 Label L_good_sp;
4452 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
4453 __ jcc(Assembler::equal, L_good_sp);
4454 __ stop("Incorrect rsp after prepare thaw");
4455 __ BIND(L_good_sp);
4456 }
4457 #endif // ASSERT
4458
4459 // rbx contains the size of the frames to thaw, 0 if overflow or no more frames
4460 Label L_thaw_success;
4461 __ testptr(rbx, rbx);
4462 __ jccb(Assembler::notZero, L_thaw_success);
4463 __ jump(RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
4464 __ bind(L_thaw_success);
4465
4466 // Make room for the thawed frames and align the stack.
4467 __ subptr(rsp, rbx);
4468 __ andptr(rsp, -StackAlignmentInBytes);
4469
4470 if (return_barrier) {
4471 // Preserve possible return value from a method returning to the return barrier. (Again.)
4472 __ push_ppx(rax);
4473 __ push_d(xmm0);
4474 }
4475
4476 // If we want, we can templatize thaw by kind, and have three different entries.
4477 __ movptr(c_rarg0, r15_thread);
4478 __ movptr(c_rarg1, kind);
4479 __ call_VM_leaf(Continuation::thaw_entry(), 2);
4480 __ movptr(rbx, rax);
4481
4482 if (return_barrier) {
4483 // Restore return value from a method returning to the return barrier. (Again.)
4484 // No safepoint in the call to thaw, so even an oop return value should be OK.
4485 __ pop_d(xmm0);
4486 __ pop_ppx(rax);
4487 } else {
4488 // Return 0 (success) from doYield.
4489 __ xorptr(rax, rax);
4490 }
4491
4492 // After thawing, rbx is the SP of the yielding frame.
4493 // Move there, and then to saved RBP slot.
4494 __ movptr(rsp, rbx);
4495 __ subptr(rsp, 2*wordSize);
4496
4497 if (return_barrier_exception) {
4498 __ movptr(c_rarg0, r15_thread);
4499 __ movptr(c_rarg1, Address(rsp, wordSize)); // return address
4500
4501 // rax still holds the original exception oop, save it before the call
4502 __ push_ppx(rax);
4503
4504 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), 2);
4505 __ movptr(rbx, rax);
4506
4733
4734 void StubGenerator::generate_initial_stubs() {
4735 // Generates all stubs and initializes the entry points
4736
4737 // This platform-specific settings are needed by generate_call_stub()
4738 create_control_words();
4739
4740 // Initialize table for unsafe copy memeory check.
4741 if (UnsafeMemoryAccess::_table == nullptr) {
4742 UnsafeMemoryAccess::create_table(16 + 4); // 16 for copyMemory; 4 for setMemory
4743 }
4744
4745 // entry points that exist in all platforms Note: This is code
4746 // that could be shared among different platforms - however the
4747 // benefit seems to be smaller than the disadvantage of having a
4748 // much more complicated generator structure. See also comment in
4749 // stubRoutines.hpp.
4750
4751 StubRoutines::_forward_exception_entry = generate_forward_exception();
4752
4753 StubRoutines::_call_stub_entry =
4754 generate_call_stub(StubRoutines::_call_stub_return_address);
4755
4756 // is referenced by megamorphic call
4757 StubRoutines::_catch_exception_entry = generate_catch_exception();
4758
4759 // platform dependent
4760 StubRoutines::x86::_verify_mxcsr_entry = generate_verify_mxcsr();
4761
4762 StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
4763 StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
4764 StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
4765 StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
4766
4767 StubRoutines::x86::_float_sign_mask = generate_fp_mask(StubId::stubgen_float_sign_mask_id, 0x7FFFFFFF7FFFFFFF);
4768 StubRoutines::x86::_float_sign_flip = generate_fp_mask(StubId::stubgen_float_sign_flip_id, 0x8000000080000000);
4769 StubRoutines::x86::_double_sign_mask = generate_fp_mask(StubId::stubgen_double_sign_mask_id, 0x7FFFFFFFFFFFFFFF);
4770 StubRoutines::x86::_double_sign_flip = generate_fp_mask(StubId::stubgen_double_sign_flip_id, 0x8000000000000000);
4771
4772 if (UseCRC32Intrinsics) {
4776 if (UseCRC32CIntrinsics) {
4777 bool supports_clmul = VM_Version::supports_clmul();
4778 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
4779 }
4780
4781 if (VM_Version::supports_float16()) {
4782 // For results consistency both intrinsics should be enabled.
4783 // vmIntrinsics checks InlineIntrinsics flag, no need to check it here.
4784 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
4785 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
4786 StubRoutines::_hf2f = generate_float16ToFloat();
4787 StubRoutines::_f2hf = generate_floatToFloat16();
4788 }
4789 }
4790
4791 generate_libm_stubs();
4792
4793 StubRoutines::_fmod = generate_libmFmod(); // from stubGenerator_x86_64_fmod.cpp
4794 }
4795
4796 void StubGenerator::generate_continuation_stubs() {
4797 // Continuation stubs:
4798 StubRoutines::_cont_thaw = generate_cont_thaw();
4799 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
4800 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
4801 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
4802 }
4803
4804 void StubGenerator::generate_final_stubs() {
4805 // Generates the rest of stubs and initializes the entry points
4806
4807 // support for verify_oop (must happen after universe_init)
4808 if (VerifyOops) {
4809 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
4810 }
4811
4812 // arraycopy stubs used by compilers
4813 generate_arraycopy_stubs();
4814
4815 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
|
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/assembler.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "classfile/vmIntrinsics.hpp"
29 #include "compiler/oopMap.hpp"
30 #include "gc/shared/barrierSet.hpp"
31 #include "gc/shared/barrierSetAssembler.hpp"
32 #include "gc/shared/barrierSetNMethod.hpp"
33 #include "gc/shared/gc_globals.hpp"
34 #include "memory/universe.hpp"
35 #include "oops/inlineKlass.hpp"
36 #include "prims/jvmtiExport.hpp"
37 #include "prims/upcallLinker.hpp"
38 #include "runtime/arguments.hpp"
39 #include "runtime/continuationEntry.hpp"
40 #include "runtime/javaThread.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "runtime/stubRoutines.hpp"
43 #include "utilities/macros.hpp"
44 #include "vmreg_x86.inline.hpp"
45 #include "stubGenerator_x86_64.hpp"
46 #ifdef COMPILER2
47 #include "opto/runtime.hpp"
48 #include "opto/c2_globals.hpp"
49 #endif
50 #if INCLUDE_JVMCI
51 #include "jvmci/jvmci_globals.hpp"
52 #endif
53
54 // For a more detailed description of the stub routine structure
55 // see the comment in stubRoutines.hpp
56
57 #define __ _masm->
58 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
59
60 #ifdef PRODUCT
61 #define BLOCK_COMMENT(str) /* nothing */
62 #else
63 #define BLOCK_COMMENT(str) __ block_comment(str)
64 #endif // PRODUCT
299 __ movptr(rax, Address(c_rarg2, 0));// get parameter
300 __ addptr(c_rarg2, wordSize); // advance to next parameter
301 __ decrementl(c_rarg1); // decrement counter
302 __ push(rax); // pass parameter
303 __ jcc(Assembler::notZero, loop);
304
305 // call Java function
306 __ BIND(parameters_done);
307 __ movptr(rbx, method); // get Method*
308 __ movptr(c_rarg1, entry_point); // get entry_point
309 __ mov(r13, rsp); // set sender sp
310 BLOCK_COMMENT("call Java function");
311 __ call(c_rarg1);
312
313 BLOCK_COMMENT("call_stub_return_address:");
314 return_address = __ pc();
315 entries.append(return_address);
316
317 // store result depending on type (everything that is not
318 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
319 __ movptr(r13, result);
320 Label is_long, is_float, is_double, check_prim, exit;
321 __ movl(rbx, result_type);
322 __ cmpl(rbx, T_OBJECT);
323 __ jcc(Assembler::equal, check_prim);
324 __ cmpl(rbx, T_LONG);
325 __ jcc(Assembler::equal, is_long);
326 __ cmpl(rbx, T_FLOAT);
327 __ jcc(Assembler::equal, is_float);
328 __ cmpl(rbx, T_DOUBLE);
329 __ jcc(Assembler::equal, is_double);
330 #ifdef ASSERT
331 // make sure the type is INT
332 {
333 Label L;
334 __ cmpl(rbx, T_INT);
335 __ jcc(Assembler::equal, L);
336 __ stop("StubRoutines::call_stub: unexpected result type");
337 __ bind(L);
338 }
339 #endif
340
341 // handle T_INT case
342 __ movl(Address(r13, 0), rax);
343
344 __ BIND(exit);
345
346 // pop parameters
347 __ lea(rsp, rsp_after_call);
348
349 #ifdef ASSERT
350 // verify that threads correspond
351 {
352 Label L1, L2, L3;
353 __ cmpptr(r15_thread, thread);
354 __ jcc(Assembler::equal, L1);
355 __ stop("StubRoutines::call_stub: r15_thread is corrupted");
356 __ bind(L1);
357 __ get_thread_slow(rbx);
358 __ cmpptr(r15_thread, thread);
359 __ jcc(Assembler::equal, L2);
360 __ stop("StubRoutines::call_stub: r15_thread is modified by call");
361 __ bind(L2);
362 __ cmpptr(r15_thread, rbx);
380 __ movptr(r13, r13_save);
381 __ movptr(r12, r12_save);
382 __ movptr(rbx, rbx_save);
383
384 #ifdef _WIN64
385 __ movptr(rdi, rdi_save);
386 __ movptr(rsi, rsi_save);
387 #else
388 __ ldmxcsr(mxcsr_save);
389 #endif
390
391 // restore rsp
392 __ addptr(rsp, -rsp_after_call_off * wordSize);
393
394 // return
395 __ vzeroupper();
396 __ pop(rbp);
397 __ ret(0);
398
399 // handle return types different from T_INT
400 __ BIND(check_prim);
401 if (InlineTypeReturnedAsFields) {
402 // Check for scalarized return value
403 __ testptr(rax, 1);
404 __ jcc(Assembler::zero, is_long);
405 // Load pack handler address
406 __ andptr(rax, -2);
407 __ movptr(rax, Address(rax, InlineKlass::adr_members_offset()));
408 __ movptr(rbx, Address(rax, InlineKlass::pack_handler_jobject_offset()));
409 // Call pack handler to initialize the buffer
410 __ call(rbx);
411 __ jmp(exit);
412 }
413 __ BIND(is_long);
414 __ movq(Address(r13, 0), rax);
415 __ jmp(exit);
416
417 __ BIND(is_float);
418 __ movflt(Address(r13, 0), xmm0);
419 __ jmp(exit);
420
421 __ BIND(is_double);
422 __ movdbl(Address(r13, 0), xmm0);
423 __ jmp(exit);
424
425 // record the stub entry and end plus the auxiliary entry
426 store_archive_data(stub_id, start, __ pc(), &entries);
427
428 return start;
429 }
430
431 // Return point for a Java call if there's an exception thrown in
432 // Java code. The exception is caught and transformed into a
433 // pending exception stored in JavaThread that can be tested from
434 // within the VM.
435 //
436 // Note: Usually the parameters are removed by the callee. In case
437 // of an exception crossing an activation frame boundary, that is
438 // not the case if the callee is compiled code => need to setup the
439 // rsp.
440 //
441 // rax: exception oop
442
4373 return start;
4374 }
4375 StubCodeMark mark(this, stub_id);
4376
4377 start = __ pc();
4378
4379 BLOCK_COMMENT("Entry:");
4380 // No need for RuntimeStub frame since it is called only during JIT compilation
4381
4382 // Convert and put result into rax
4383 __ flt_to_flt16(rax, xmm0, xmm1);
4384
4385 __ ret(0);
4386
4387 // record the stub entry and end
4388 store_archive_data(stub_id, start, __ pc());
4389
4390 return start;
4391 }
4392
4393 static void save_return_registers(MacroAssembler* masm) {
4394 masm->push_ppx(rax);
4395 if (InlineTypeReturnedAsFields) {
4396 masm->push(rdi);
4397 masm->push(rsi);
4398 masm->push(rdx);
4399 masm->push(rcx);
4400 masm->push(r8);
4401 masm->push(r9);
4402 }
4403 masm->push_d(xmm0);
4404 if (InlineTypeReturnedAsFields) {
4405 masm->push_d(xmm1);
4406 masm->push_d(xmm2);
4407 masm->push_d(xmm3);
4408 masm->push_d(xmm4);
4409 masm->push_d(xmm5);
4410 masm->push_d(xmm6);
4411 masm->push_d(xmm7);
4412 }
4413 #ifdef ASSERT
4414 masm->movq(rax, 0xBADC0FFE);
4415 masm->movq(rdi, rax);
4416 masm->movq(rsi, rax);
4417 masm->movq(rdx, rax);
4418 masm->movq(rcx, rax);
4419 masm->movq(r8, rax);
4420 masm->movq(r9, rax);
4421 masm->movq(xmm0, rax);
4422 masm->movq(xmm1, rax);
4423 masm->movq(xmm2, rax);
4424 masm->movq(xmm3, rax);
4425 masm->movq(xmm4, rax);
4426 masm->movq(xmm5, rax);
4427 masm->movq(xmm6, rax);
4428 masm->movq(xmm7, rax);
4429 #endif
4430 }
4431
4432 static void restore_return_registers(MacroAssembler* masm) {
4433 if (InlineTypeReturnedAsFields) {
4434 masm->pop_d(xmm7);
4435 masm->pop_d(xmm6);
4436 masm->pop_d(xmm5);
4437 masm->pop_d(xmm4);
4438 masm->pop_d(xmm3);
4439 masm->pop_d(xmm2);
4440 masm->pop_d(xmm1);
4441 }
4442 masm->pop_d(xmm0);
4443 if (InlineTypeReturnedAsFields) {
4444 masm->pop(r9);
4445 masm->pop(r8);
4446 masm->pop(rcx);
4447 masm->pop(rdx);
4448 masm->pop(rsi);
4449 masm->pop(rdi);
4450 }
4451 masm->pop_ppx(rax);
4452 }
4453
4454 address StubGenerator::generate_cont_thaw(StubId stub_id) {
4455 if (!Continuations::enabled()) return nullptr;
4456
4457 bool return_barrier;
4458 bool return_barrier_exception;
4459 Continuation::thaw_kind kind;
4460
4461 switch (stub_id) {
4462 case StubId::stubgen_cont_thaw_id:
4463 return_barrier = false;
4464 return_barrier_exception = false;
4465 kind = Continuation::thaw_top;
4466 break;
4467 case StubId::stubgen_cont_returnBarrier_id:
4468 return_barrier = true;
4469 return_barrier_exception = false;
4470 kind = Continuation::thaw_return_barrier;
4471 break;
4472 case StubId::stubgen_cont_returnBarrierExc_id:
4473 return_barrier = true;
4491 if (!return_barrier) {
4492 // Pop return address. If we don't do this, we get a drift,
4493 // where the bottom-most frozen frame continuously grows.
4494 __ pop(c_rarg3);
4495 } else {
4496 __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
4497 }
4498
4499 #ifdef ASSERT
4500 {
4501 Label L_good_sp;
4502 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
4503 __ jcc(Assembler::equal, L_good_sp);
4504 __ stop("Incorrect rsp at thaw entry");
4505 __ BIND(L_good_sp);
4506 }
4507 #endif // ASSERT
4508
4509 if (return_barrier) {
4510 // Preserve possible return value from a method returning to the return barrier.
4511 save_return_registers(_masm);
4512 }
4513
4514 __ movptr(c_rarg0, r15_thread);
4515 __ movptr(c_rarg1, (return_barrier ? 1 : 0));
4516 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), 2);
4517 __ movptr(rbx, rax);
4518
4519 if (return_barrier) {
4520 // Restore return value from a method returning to the return barrier.
4521 // No safepoint in the call to thaw, so even an oop return value should be OK.
4522 restore_return_registers(_masm);
4523 }
4524
4525 #ifdef ASSERT
4526 {
4527 Label L_good_sp;
4528 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
4529 __ jcc(Assembler::equal, L_good_sp);
4530 __ stop("Incorrect rsp after prepare thaw");
4531 __ BIND(L_good_sp);
4532 }
4533 #endif // ASSERT
4534
4535 // rbx contains the size of the frames to thaw, 0 if overflow or no more frames
4536 Label L_thaw_success;
4537 __ testptr(rbx, rbx);
4538 __ jccb(Assembler::notZero, L_thaw_success);
4539 __ jump(RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
4540 __ bind(L_thaw_success);
4541
4542 // Make room for the thawed frames and align the stack.
4543 __ subptr(rsp, rbx);
4544 __ andptr(rsp, -StackAlignmentInBytes);
4545
4546 if (return_barrier) {
4547 // Preserve possible return value from a method returning to the return barrier. (Again.)
4548 save_return_registers(_masm);
4549 }
4550
4551 // If we want, we can templatize thaw by kind, and have three different entries.
4552 __ movptr(c_rarg0, r15_thread);
4553 __ movptr(c_rarg1, kind);
4554 __ call_VM_leaf(Continuation::thaw_entry(), 2);
4555 __ movptr(rbx, rax);
4556
4557 if (return_barrier) {
4558 // Restore return value from a method returning to the return barrier. (Again.)
4559 // No safepoint in the call to thaw, so even an oop return value should be OK.
4560 restore_return_registers(_masm);
4561 } else {
4562 // Return 0 (success) from doYield.
4563 __ xorptr(rax, rax);
4564 }
4565
4566 // After thawing, rbx is the SP of the yielding frame.
4567 // Move there, and then to saved RBP slot.
4568 __ movptr(rsp, rbx);
4569 __ subptr(rsp, 2*wordSize);
4570
4571 if (return_barrier_exception) {
4572 __ movptr(c_rarg0, r15_thread);
4573 __ movptr(c_rarg1, Address(rsp, wordSize)); // return address
4574
4575 // rax still holds the original exception oop, save it before the call
4576 __ push_ppx(rax);
4577
4578 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), 2);
4579 __ movptr(rbx, rax);
4580
4807
4808 void StubGenerator::generate_initial_stubs() {
4809 // Generates all stubs and initializes the entry points
4810
4811 // This platform-specific settings are needed by generate_call_stub()
4812 create_control_words();
4813
4814 // Initialize table for unsafe copy memeory check.
4815 if (UnsafeMemoryAccess::_table == nullptr) {
4816 UnsafeMemoryAccess::create_table(16 + 4); // 16 for copyMemory; 4 for setMemory
4817 }
4818
4819 // entry points that exist in all platforms Note: This is code
4820 // that could be shared among different platforms - however the
4821 // benefit seems to be smaller than the disadvantage of having a
4822 // much more complicated generator structure. See also comment in
4823 // stubRoutines.hpp.
4824
4825 StubRoutines::_forward_exception_entry = generate_forward_exception();
4826
4827 // Generate these first because they are called from other stubs
4828 if (InlineTypeReturnedAsFields) {
4829 StubRoutines::_load_inline_type_fields_in_regs =
4830 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::load_inline_type_fields_in_regs),
4831 "load_inline_type_fields_in_regs", false);
4832 StubRoutines::_store_inline_type_fields_to_buf =
4833 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::store_inline_type_fields_to_buf),
4834 "store_inline_type_fields_to_buf", true);
4835 }
4836
4837 StubRoutines::_call_stub_entry =
4838 generate_call_stub(StubRoutines::_call_stub_return_address);
4839
4840 // is referenced by megamorphic call
4841 StubRoutines::_catch_exception_entry = generate_catch_exception();
4842
4843 // platform dependent
4844 StubRoutines::x86::_verify_mxcsr_entry = generate_verify_mxcsr();
4845
4846 StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
4847 StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
4848 StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
4849 StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
4850
4851 StubRoutines::x86::_float_sign_mask = generate_fp_mask(StubId::stubgen_float_sign_mask_id, 0x7FFFFFFF7FFFFFFF);
4852 StubRoutines::x86::_float_sign_flip = generate_fp_mask(StubId::stubgen_float_sign_flip_id, 0x8000000080000000);
4853 StubRoutines::x86::_double_sign_mask = generate_fp_mask(StubId::stubgen_double_sign_mask_id, 0x7FFFFFFFFFFFFFFF);
4854 StubRoutines::x86::_double_sign_flip = generate_fp_mask(StubId::stubgen_double_sign_flip_id, 0x8000000000000000);
4855
4856 if (UseCRC32Intrinsics) {
4860 if (UseCRC32CIntrinsics) {
4861 bool supports_clmul = VM_Version::supports_clmul();
4862 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
4863 }
4864
4865 if (VM_Version::supports_float16()) {
4866 // For results consistency both intrinsics should be enabled.
4867 // vmIntrinsics checks InlineIntrinsics flag, no need to check it here.
4868 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
4869 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
4870 StubRoutines::_hf2f = generate_float16ToFloat();
4871 StubRoutines::_f2hf = generate_floatToFloat16();
4872 }
4873 }
4874
4875 generate_libm_stubs();
4876
4877 StubRoutines::_fmod = generate_libmFmod(); // from stubGenerator_x86_64_fmod.cpp
4878 }
4879
4880 // Call here from the interpreter or compiled code to either load
4881 // multiple returned values from the inline type instance being
4882 // returned to registers or to store returned values to a newly
4883 // allocated inline type instance.
4884 // Register is a class, but it would be assigned numerical value.
4885 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull.
4886 PRAGMA_DIAG_PUSH
4887 PRAGMA_NONNULL_IGNORED
4888 address StubGenerator::generate_return_value_stub(address destination, const char* name, bool has_res) {
4889 // We need to save all registers the calling convention may use so
4890 // the runtime calls read or update those registers. This needs to
4891 // be in sync with SharedRuntime::java_return_convention().
4892 enum layout {
4893 pad_off = frame::arg_reg_save_area_bytes/BytesPerInt, pad_off_2,
4894 rax_off, rax_off_2,
4895 j_rarg5_off, j_rarg5_2,
4896 j_rarg4_off, j_rarg4_2,
4897 j_rarg3_off, j_rarg3_2,
4898 j_rarg2_off, j_rarg2_2,
4899 j_rarg1_off, j_rarg1_2,
4900 j_rarg0_off, j_rarg0_2,
4901 j_farg0_off, j_farg0_2,
4902 j_farg1_off, j_farg1_2,
4903 j_farg2_off, j_farg2_2,
4904 j_farg3_off, j_farg3_2,
4905 j_farg4_off, j_farg4_2,
4906 j_farg5_off, j_farg5_2,
4907 j_farg6_off, j_farg6_2,
4908 j_farg7_off, j_farg7_2,
4909 rbp_off, rbp_off_2,
4910 return_off, return_off_2,
4911
4912 framesize
4913 };
4914
4915 CodeBuffer buffer(name, 1000, 512);
4916 MacroAssembler* _masm = new MacroAssembler(&buffer);
4917
4918 int frame_size_in_bytes = align_up(framesize*BytesPerInt, 16);
4919 assert(frame_size_in_bytes == framesize*BytesPerInt, "misaligned");
4920 int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
4921 int frame_size_in_words = frame_size_in_bytes / wordSize;
4922
4923 OopMapSet *oop_maps = new OopMapSet();
4924 OopMap* map = new OopMap(frame_size_in_slots, 0);
4925
4926 map->set_callee_saved(VMRegImpl::stack2reg(rax_off), rax->as_VMReg());
4927 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg5_off), j_rarg5->as_VMReg());
4928 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg4_off), j_rarg4->as_VMReg());
4929 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg3_off), j_rarg3->as_VMReg());
4930 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg2_off), j_rarg2->as_VMReg());
4931 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg1_off), j_rarg1->as_VMReg());
4932 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg0_off), j_rarg0->as_VMReg());
4933 map->set_callee_saved(VMRegImpl::stack2reg(j_farg0_off), j_farg0->as_VMReg());
4934 map->set_callee_saved(VMRegImpl::stack2reg(j_farg1_off), j_farg1->as_VMReg());
4935 map->set_callee_saved(VMRegImpl::stack2reg(j_farg2_off), j_farg2->as_VMReg());
4936 map->set_callee_saved(VMRegImpl::stack2reg(j_farg3_off), j_farg3->as_VMReg());
4937 map->set_callee_saved(VMRegImpl::stack2reg(j_farg4_off), j_farg4->as_VMReg());
4938 map->set_callee_saved(VMRegImpl::stack2reg(j_farg5_off), j_farg5->as_VMReg());
4939 map->set_callee_saved(VMRegImpl::stack2reg(j_farg6_off), j_farg6->as_VMReg());
4940 map->set_callee_saved(VMRegImpl::stack2reg(j_farg7_off), j_farg7->as_VMReg());
4941
4942 int start = __ offset();
4943
4944 __ subptr(rsp, frame_size_in_bytes - 8 /* return address*/);
4945
4946 __ movptr(Address(rsp, rbp_off * BytesPerInt), rbp);
4947 __ movdbl(Address(rsp, j_farg7_off * BytesPerInt), j_farg7);
4948 __ movdbl(Address(rsp, j_farg6_off * BytesPerInt), j_farg6);
4949 __ movdbl(Address(rsp, j_farg5_off * BytesPerInt), j_farg5);
4950 __ movdbl(Address(rsp, j_farg4_off * BytesPerInt), j_farg4);
4951 __ movdbl(Address(rsp, j_farg3_off * BytesPerInt), j_farg3);
4952 __ movdbl(Address(rsp, j_farg2_off * BytesPerInt), j_farg2);
4953 __ movdbl(Address(rsp, j_farg1_off * BytesPerInt), j_farg1);
4954 __ movdbl(Address(rsp, j_farg0_off * BytesPerInt), j_farg0);
4955
4956 __ movptr(Address(rsp, j_rarg0_off * BytesPerInt), j_rarg0);
4957 __ movptr(Address(rsp, j_rarg1_off * BytesPerInt), j_rarg1);
4958 __ movptr(Address(rsp, j_rarg2_off * BytesPerInt), j_rarg2);
4959 __ movptr(Address(rsp, j_rarg3_off * BytesPerInt), j_rarg3);
4960 __ movptr(Address(rsp, j_rarg4_off * BytesPerInt), j_rarg4);
4961 __ movptr(Address(rsp, j_rarg5_off * BytesPerInt), j_rarg5);
4962 __ movptr(Address(rsp, rax_off * BytesPerInt), rax);
4963
4964 int frame_complete = __ offset();
4965
4966 __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1);
4967
4968 __ mov(c_rarg0, r15_thread);
4969 __ mov(c_rarg1, rax);
4970
4971 __ call(RuntimeAddress(destination));
4972
4973 // Set an oopmap for the call site.
4974
4975 oop_maps->add_gc_map( __ offset() - start, map);
4976
4977 // clear last_Java_sp
4978 __ reset_last_Java_frame(false);
4979
4980 __ movptr(rbp, Address(rsp, rbp_off * BytesPerInt));
4981 __ movdbl(j_farg7, Address(rsp, j_farg7_off * BytesPerInt));
4982 __ movdbl(j_farg6, Address(rsp, j_farg6_off * BytesPerInt));
4983 __ movdbl(j_farg5, Address(rsp, j_farg5_off * BytesPerInt));
4984 __ movdbl(j_farg4, Address(rsp, j_farg4_off * BytesPerInt));
4985 __ movdbl(j_farg3, Address(rsp, j_farg3_off * BytesPerInt));
4986 __ movdbl(j_farg2, Address(rsp, j_farg2_off * BytesPerInt));
4987 __ movdbl(j_farg1, Address(rsp, j_farg1_off * BytesPerInt));
4988 __ movdbl(j_farg0, Address(rsp, j_farg0_off * BytesPerInt));
4989
4990 __ movptr(j_rarg0, Address(rsp, j_rarg0_off * BytesPerInt));
4991 __ movptr(j_rarg1, Address(rsp, j_rarg1_off * BytesPerInt));
4992 __ movptr(j_rarg2, Address(rsp, j_rarg2_off * BytesPerInt));
4993 __ movptr(j_rarg3, Address(rsp, j_rarg3_off * BytesPerInt));
4994 __ movptr(j_rarg4, Address(rsp, j_rarg4_off * BytesPerInt));
4995 __ movptr(j_rarg5, Address(rsp, j_rarg5_off * BytesPerInt));
4996 __ movptr(rax, Address(rsp, rax_off * BytesPerInt));
4997
4998 __ addptr(rsp, frame_size_in_bytes-8);
4999
5000 // check for pending exceptions
5001 Label pending;
5002 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
5003 __ jcc(Assembler::notEqual, pending);
5004
5005 if (has_res) {
5006 // We just called SharedRuntime::store_inline_type_fields_to_buf. Check if we still
5007 // need to initialize the buffer and if so, call the inline class specific pack handler.
5008 Label skip_pack;
5009 __ get_vm_result_oop(rax);
5010 __ get_vm_result_metadata(rscratch1);
5011 __ testptr(rscratch1, rscratch1);
5012 __ jcc(Assembler::zero, skip_pack);
5013 __ movptr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
5014 __ movptr(rscratch1, Address(rscratch1, InlineKlass::pack_handler_offset()));
5015 __ call(rscratch1);
5016 __ membar(Assembler::StoreStore);
5017 __ bind(skip_pack);
5018 }
5019
5020 __ ret(0);
5021
5022 __ bind(pending);
5023
5024 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
5025 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
5026
5027 // -------------
5028 // make sure all code is generated
5029 _masm->flush();
5030
5031 RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, false);
5032 return stub->entry_point();
5033 }
5034
5035 void StubGenerator::generate_continuation_stubs() {
5036 // Continuation stubs:
5037 StubRoutines::_cont_thaw = generate_cont_thaw();
5038 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
5039 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
5040 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
5041 }
5042
5043 void StubGenerator::generate_final_stubs() {
5044 // Generates the rest of stubs and initializes the entry points
5045
5046 // support for verify_oop (must happen after universe_init)
5047 if (VerifyOops) {
5048 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
5049 }
5050
5051 // arraycopy stubs used by compilers
5052 generate_arraycopy_stubs();
5053
5054 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
|