5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "classfile/javaClasses.hpp"
27 #include "classfile/vmIntrinsics.hpp"
28 #include "compiler/oopMap.hpp"
29 #include "gc/shared/barrierSet.hpp"
30 #include "gc/shared/barrierSetAssembler.hpp"
31 #include "gc/shared/barrierSetNMethod.hpp"
32 #include "gc/shared/gc_globals.hpp"
33 #include "memory/universe.hpp"
34 #include "prims/jvmtiExport.hpp"
35 #include "prims/upcallLinker.hpp"
36 #include "runtime/arguments.hpp"
37 #include "runtime/continuationEntry.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #include "runtime/stubRoutines.hpp"
41 #include "stubGenerator_x86_64.hpp"
42 #ifdef COMPILER2
43 #include "opto/runtime.hpp"
44 #include "opto/c2_globals.hpp"
45 #endif
46
47 // For a more detailed description of the stub routine structure
48 // see the comment in stubRoutines.hpp
49
50 #define __ _masm->
51 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
52
53 #ifdef PRODUCT
54 #define BLOCK_COMMENT(str) /* nothing */
55 #else
56 #define BLOCK_COMMENT(str) __ block_comment(str)
57 #endif // PRODUCT
58
59 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
60
292 __ movptr(rax, Address(c_rarg2, 0));// get parameter
293 __ addptr(c_rarg2, wordSize); // advance to next parameter
294 __ decrementl(c_rarg1); // decrement counter
295 __ push(rax); // pass parameter
296 __ jcc(Assembler::notZero, loop);
297
298 // call Java function
299 __ BIND(parameters_done);
300 __ movptr(rbx, method); // get Method*
301 __ movptr(c_rarg1, entry_point); // get entry_point
302 __ mov(r13, rsp); // set sender sp
303 BLOCK_COMMENT("call Java function");
304 __ call(c_rarg1);
305
306 BLOCK_COMMENT("call_stub_return_address:");
307 return_address = __ pc();
308 entries.append(return_address);
309
310 // store result depending on type (everything that is not
311 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
312 __ movptr(c_rarg0, result);
313 Label is_long, is_float, is_double, exit;
314 __ movl(c_rarg1, result_type);
315 __ cmpl(c_rarg1, T_OBJECT);
316 __ jcc(Assembler::equal, is_long);
317 __ cmpl(c_rarg1, T_LONG);
318 __ jcc(Assembler::equal, is_long);
319 __ cmpl(c_rarg1, T_FLOAT);
320 __ jcc(Assembler::equal, is_float);
321 __ cmpl(c_rarg1, T_DOUBLE);
322 __ jcc(Assembler::equal, is_double);
323 #ifdef ASSERT
324 // make sure the type is INT
325 {
326 Label L;
327 __ cmpl(c_rarg1, T_INT);
328 __ jcc(Assembler::equal, L);
329 __ stop("StubRoutines::call_stub: unexpected result type");
330 __ bind(L);
331 }
332 #endif
333
334 // handle T_INT case
335 __ movl(Address(c_rarg0, 0), rax);
336
337 __ BIND(exit);
338
339 // pop parameters
340 __ lea(rsp, rsp_after_call);
341
342 #ifdef ASSERT
343 // verify that threads correspond
344 {
345 Label L1, L2, L3;
346 __ cmpptr(r15_thread, thread);
347 __ jcc(Assembler::equal, L1);
348 __ stop("StubRoutines::call_stub: r15_thread is corrupted");
349 __ bind(L1);
350 __ get_thread_slow(rbx);
351 __ cmpptr(r15_thread, thread);
352 __ jcc(Assembler::equal, L2);
353 __ stop("StubRoutines::call_stub: r15_thread is modified by call");
354 __ bind(L2);
355 __ cmpptr(r15_thread, rbx);
373 __ movptr(r13, r13_save);
374 __ movptr(r12, r12_save);
375 __ movptr(rbx, rbx_save);
376
377 #ifdef _WIN64
378 __ movptr(rdi, rdi_save);
379 __ movptr(rsi, rsi_save);
380 #else
381 __ ldmxcsr(mxcsr_save);
382 #endif
383
384 // restore rsp
385 __ addptr(rsp, -rsp_after_call_off * wordSize);
386
387 // return
388 __ vzeroupper();
389 __ pop(rbp);
390 __ ret(0);
391
392 // handle return types different from T_INT
393 __ BIND(is_long);
394 __ movq(Address(c_rarg0, 0), rax);
395 __ jmp(exit);
396
397 __ BIND(is_float);
398 __ movflt(Address(c_rarg0, 0), xmm0);
399 __ jmp(exit);
400
401 __ BIND(is_double);
402 __ movdbl(Address(c_rarg0, 0), xmm0);
403 __ jmp(exit);
404
405 // record the stub entry and end plus the auxiliary entry
406 store_archive_data(stub_id, start, __ pc(), &entries);
407
408 return start;
409 }
410
411 // Return point for a Java call if there's an exception thrown in
412 // Java code. The exception is caught and transformed into a
413 // pending exception stored in JavaThread that can be tested from
414 // within the VM.
415 //
416 // Note: Usually the parameters are removed by the callee. In case
417 // of an exception crossing an activation frame boundary, that is
418 // not the case if the callee is compiled code => need to setup the
419 // rsp.
420 //
421 // rax: exception oop
422
4353 return start;
4354 }
4355 StubCodeMark mark(this, stub_id);
4356
4357 start = __ pc();
4358
4359 BLOCK_COMMENT("Entry:");
4360 // No need for RuntimeStub frame since it is called only during JIT compilation
4361
4362 // Convert and put result into rax
4363 __ flt_to_flt16(rax, xmm0, xmm1);
4364
4365 __ ret(0);
4366
4367 // record the stub entry and end
4368 store_archive_data(stub_id, start, __ pc());
4369
4370 return start;
4371 }
4372
4373 address StubGenerator::generate_cont_thaw(StubId stub_id) {
4374 if (!Continuations::enabled()) return nullptr;
4375
4376 bool return_barrier;
4377 bool return_barrier_exception;
4378 Continuation::thaw_kind kind;
4379
4380 switch (stub_id) {
4381 case StubId::stubgen_cont_thaw_id:
4382 return_barrier = false;
4383 return_barrier_exception = false;
4384 kind = Continuation::thaw_top;
4385 break;
4386 case StubId::stubgen_cont_returnBarrier_id:
4387 return_barrier = true;
4388 return_barrier_exception = false;
4389 kind = Continuation::thaw_return_barrier;
4390 break;
4391 case StubId::stubgen_cont_returnBarrierExc_id:
4392 return_barrier = true;
4410 if (!return_barrier) {
4411 // Pop return address. If we don't do this, we get a drift,
4412 // where the bottom-most frozen frame continuously grows.
4413 __ pop(c_rarg3);
4414 } else {
4415 __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
4416 }
4417
4418 #ifdef ASSERT
4419 {
4420 Label L_good_sp;
4421 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
4422 __ jcc(Assembler::equal, L_good_sp);
4423 __ stop("Incorrect rsp at thaw entry");
4424 __ BIND(L_good_sp);
4425 }
4426 #endif // ASSERT
4427
4428 if (return_barrier) {
4429 // Preserve possible return value from a method returning to the return barrier.
4430 __ push_ppx(rax);
4431 __ push_d(xmm0);
4432 }
4433
4434 __ movptr(c_rarg0, r15_thread);
4435 __ movptr(c_rarg1, (return_barrier ? 1 : 0));
4436 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), 2);
4437 __ movptr(rbx, rax);
4438
4439 if (return_barrier) {
4440 // Restore return value from a method returning to the return barrier.
4441 // No safepoint in the call to thaw, so even an oop return value should be OK.
4442 __ pop_d(xmm0);
4443 __ pop_ppx(rax);
4444 }
4445
4446 #ifdef ASSERT
4447 {
4448 Label L_good_sp;
4449 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
4450 __ jcc(Assembler::equal, L_good_sp);
4451 __ stop("Incorrect rsp after prepare thaw");
4452 __ BIND(L_good_sp);
4453 }
4454 #endif // ASSERT
4455
4456 // rbx contains the size of the frames to thaw, 0 if overflow or no more frames
4457 Label L_thaw_success;
4458 __ testptr(rbx, rbx);
4459 __ jccb(Assembler::notZero, L_thaw_success);
4460 __ jump(RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
4461 __ bind(L_thaw_success);
4462
4463 // Make room for the thawed frames and align the stack.
4464 __ subptr(rsp, rbx);
4465 __ andptr(rsp, -StackAlignmentInBytes);
4466
4467 if (return_barrier) {
4468 // Preserve possible return value from a method returning to the return barrier. (Again.)
4469 __ push_ppx(rax);
4470 __ push_d(xmm0);
4471 }
4472
4473 // If we want, we can templatize thaw by kind, and have three different entries.
4474 __ movptr(c_rarg0, r15_thread);
4475 __ movptr(c_rarg1, kind);
4476 __ call_VM_leaf(Continuation::thaw_entry(), 2);
4477 __ movptr(rbx, rax);
4478
4479 if (return_barrier) {
4480 // Restore return value from a method returning to the return barrier. (Again.)
4481 // No safepoint in the call to thaw, so even an oop return value should be OK.
4482 __ pop_d(xmm0);
4483 __ pop_ppx(rax);
4484 } else {
4485 // Return 0 (success) from doYield.
4486 __ xorptr(rax, rax);
4487 }
4488
4489 // After thawing, rbx is the SP of the yielding frame.
4490 // Move there, and then to saved RBP slot.
4491 __ movptr(rsp, rbx);
4492 __ subptr(rsp, 2*wordSize);
4493
4494 if (return_barrier_exception) {
4495 __ movptr(c_rarg0, r15_thread);
4496 __ movptr(c_rarg1, Address(rsp, wordSize)); // return address
4497
4498 // rax still holds the original exception oop, save it before the call
4499 __ push_ppx(rax);
4500
4501 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), 2);
4502 __ movptr(rbx, rax);
4503
4730
4731 void StubGenerator::generate_initial_stubs() {
4732 // Generates all stubs and initializes the entry points
4733
4734 // This platform-specific settings are needed by generate_call_stub()
4735 create_control_words();
4736
4737 // Initialize table for unsafe copy memeory check.
4738 if (UnsafeMemoryAccess::_table == nullptr) {
4739 UnsafeMemoryAccess::create_table(16 + 4); // 16 for copyMemory; 4 for setMemory
4740 }
4741
4742 // entry points that exist in all platforms Note: This is code
4743 // that could be shared among different platforms - however the
4744 // benefit seems to be smaller than the disadvantage of having a
4745 // much more complicated generator structure. See also comment in
4746 // stubRoutines.hpp.
4747
4748 StubRoutines::_forward_exception_entry = generate_forward_exception();
4749
4750 StubRoutines::_call_stub_entry =
4751 generate_call_stub(StubRoutines::_call_stub_return_address);
4752
4753 // is referenced by megamorphic call
4754 StubRoutines::_catch_exception_entry = generate_catch_exception();
4755
4756 // platform dependent
4757 StubRoutines::x86::_verify_mxcsr_entry = generate_verify_mxcsr();
4758
4759 StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
4760 StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
4761 StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
4762 StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
4763
4764 StubRoutines::x86::_float_sign_mask = generate_fp_mask(StubId::stubgen_float_sign_mask_id, 0x7FFFFFFF7FFFFFFF);
4765 StubRoutines::x86::_float_sign_flip = generate_fp_mask(StubId::stubgen_float_sign_flip_id, 0x8000000080000000);
4766 StubRoutines::x86::_double_sign_mask = generate_fp_mask(StubId::stubgen_double_sign_mask_id, 0x7FFFFFFFFFFFFFFF);
4767 StubRoutines::x86::_double_sign_flip = generate_fp_mask(StubId::stubgen_double_sign_flip_id, 0x8000000000000000);
4768
4769 if (UseCRC32Intrinsics) {
4773 if (UseCRC32CIntrinsics) {
4774 bool supports_clmul = VM_Version::supports_clmul();
4775 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
4776 }
4777
4778 if (VM_Version::supports_float16()) {
4779 // For results consistency both intrinsics should be enabled.
4780 // vmIntrinsics checks InlineIntrinsics flag, no need to check it here.
4781 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
4782 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
4783 StubRoutines::_hf2f = generate_float16ToFloat();
4784 StubRoutines::_f2hf = generate_floatToFloat16();
4785 }
4786 }
4787
4788 generate_libm_stubs();
4789
4790 StubRoutines::_fmod = generate_libmFmod(); // from stubGenerator_x86_64_fmod.cpp
4791 }
4792
4793 void StubGenerator::generate_continuation_stubs() {
4794 // Continuation stubs:
4795 StubRoutines::_cont_thaw = generate_cont_thaw();
4796 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
4797 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
4798 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
4799 }
4800
4801 void StubGenerator::generate_final_stubs() {
4802 // Generates the rest of stubs and initializes the entry points
4803
4804 // support for verify_oop (must happen after universe_init)
4805 if (VerifyOops) {
4806 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
4807 }
4808
4809 // arraycopy stubs used by compilers
4810 generate_arraycopy_stubs();
4811
4812 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
|
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/assembler.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "classfile/vmIntrinsics.hpp"
29 #include "compiler/oopMap.hpp"
30 #include "gc/shared/barrierSet.hpp"
31 #include "gc/shared/barrierSetAssembler.hpp"
32 #include "gc/shared/barrierSetNMethod.hpp"
33 #include "gc/shared/gc_globals.hpp"
34 #include "memory/universe.hpp"
35 #include "oops/inlineKlass.hpp"
36 #include "prims/jvmtiExport.hpp"
37 #include "prims/upcallLinker.hpp"
38 #include "runtime/arguments.hpp"
39 #include "runtime/continuationEntry.hpp"
40 #include "runtime/javaThread.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "runtime/stubRoutines.hpp"
43 #include "utilities/macros.hpp"
44 #include "vmreg_x86.inline.hpp"
45 #include "stubGenerator_x86_64.hpp"
46 #ifdef COMPILER2
47 #include "opto/runtime.hpp"
48 #include "opto/c2_globals.hpp"
49 #endif
50
51 // For a more detailed description of the stub routine structure
52 // see the comment in stubRoutines.hpp
53
54 #define __ _masm->
55 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
56
57 #ifdef PRODUCT
58 #define BLOCK_COMMENT(str) /* nothing */
59 #else
60 #define BLOCK_COMMENT(str) __ block_comment(str)
61 #endif // PRODUCT
62
63 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
64
296 __ movptr(rax, Address(c_rarg2, 0));// get parameter
297 __ addptr(c_rarg2, wordSize); // advance to next parameter
298 __ decrementl(c_rarg1); // decrement counter
299 __ push(rax); // pass parameter
300 __ jcc(Assembler::notZero, loop);
301
302 // call Java function
303 __ BIND(parameters_done);
304 __ movptr(rbx, method); // get Method*
305 __ movptr(c_rarg1, entry_point); // get entry_point
306 __ mov(r13, rsp); // set sender sp
307 BLOCK_COMMENT("call Java function");
308 __ call(c_rarg1);
309
310 BLOCK_COMMENT("call_stub_return_address:");
311 return_address = __ pc();
312 entries.append(return_address);
313
314 // store result depending on type (everything that is not
315 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
316 __ movptr(r13, result);
317 Label is_long, is_float, is_double, check_prim, exit;
318 __ movl(rbx, result_type);
319 __ cmpl(rbx, T_OBJECT);
320 __ jcc(Assembler::equal, check_prim);
321 __ cmpl(rbx, T_LONG);
322 __ jcc(Assembler::equal, is_long);
323 __ cmpl(rbx, T_FLOAT);
324 __ jcc(Assembler::equal, is_float);
325 __ cmpl(rbx, T_DOUBLE);
326 __ jcc(Assembler::equal, is_double);
327 #ifdef ASSERT
328 // make sure the type is INT
329 {
330 Label L;
331 __ cmpl(rbx, T_INT);
332 __ jcc(Assembler::equal, L);
333 __ stop("StubRoutines::call_stub: unexpected result type");
334 __ bind(L);
335 }
336 #endif
337
338 // handle T_INT case
339 __ movl(Address(r13, 0), rax);
340
341 __ BIND(exit);
342
343 // pop parameters
344 __ lea(rsp, rsp_after_call);
345
346 #ifdef ASSERT
347 // verify that threads correspond
348 {
349 Label L1, L2, L3;
350 __ cmpptr(r15_thread, thread);
351 __ jcc(Assembler::equal, L1);
352 __ stop("StubRoutines::call_stub: r15_thread is corrupted");
353 __ bind(L1);
354 __ get_thread_slow(rbx);
355 __ cmpptr(r15_thread, thread);
356 __ jcc(Assembler::equal, L2);
357 __ stop("StubRoutines::call_stub: r15_thread is modified by call");
358 __ bind(L2);
359 __ cmpptr(r15_thread, rbx);
377 __ movptr(r13, r13_save);
378 __ movptr(r12, r12_save);
379 __ movptr(rbx, rbx_save);
380
381 #ifdef _WIN64
382 __ movptr(rdi, rdi_save);
383 __ movptr(rsi, rsi_save);
384 #else
385 __ ldmxcsr(mxcsr_save);
386 #endif
387
388 // restore rsp
389 __ addptr(rsp, -rsp_after_call_off * wordSize);
390
391 // return
392 __ vzeroupper();
393 __ pop(rbp);
394 __ ret(0);
395
396 // handle return types different from T_INT
397 __ BIND(check_prim);
398 if (InlineTypeReturnedAsFields) {
399 // Check for scalarized return value
400 __ testptr(rax, 1);
401 __ jcc(Assembler::zero, is_long);
402 // Load pack handler address
403 __ andptr(rax, -2);
404 __ movptr(rax, Address(rax, InlineKlass::adr_members_offset()));
405 __ movptr(rbx, Address(rax, InlineKlass::pack_handler_jobject_offset()));
406 // Call pack handler to initialize the buffer
407 __ call(rbx);
408 __ jmp(exit);
409 }
410 __ BIND(is_long);
411 __ movq(Address(r13, 0), rax);
412 __ jmp(exit);
413
414 __ BIND(is_float);
415 __ movflt(Address(r13, 0), xmm0);
416 __ jmp(exit);
417
418 __ BIND(is_double);
419 __ movdbl(Address(r13, 0), xmm0);
420 __ jmp(exit);
421
422 // record the stub entry and end plus the auxiliary entry
423 store_archive_data(stub_id, start, __ pc(), &entries);
424
425 return start;
426 }
427
428 // Return point for a Java call if there's an exception thrown in
429 // Java code. The exception is caught and transformed into a
430 // pending exception stored in JavaThread that can be tested from
431 // within the VM.
432 //
433 // Note: Usually the parameters are removed by the callee. In case
434 // of an exception crossing an activation frame boundary, that is
435 // not the case if the callee is compiled code => need to setup the
436 // rsp.
437 //
438 // rax: exception oop
439
4370 return start;
4371 }
4372 StubCodeMark mark(this, stub_id);
4373
4374 start = __ pc();
4375
4376 BLOCK_COMMENT("Entry:");
4377 // No need for RuntimeStub frame since it is called only during JIT compilation
4378
4379 // Convert and put result into rax
4380 __ flt_to_flt16(rax, xmm0, xmm1);
4381
4382 __ ret(0);
4383
4384 // record the stub entry and end
4385 store_archive_data(stub_id, start, __ pc());
4386
4387 return start;
4388 }
4389
4390 static void save_return_registers(MacroAssembler* masm) {
4391 masm->push_ppx(rax);
4392 if (InlineTypeReturnedAsFields) {
4393 masm->push(rdi);
4394 masm->push(rsi);
4395 masm->push(rdx);
4396 masm->push(rcx);
4397 masm->push(r8);
4398 masm->push(r9);
4399 }
4400 masm->push_d(xmm0);
4401 if (InlineTypeReturnedAsFields) {
4402 masm->push_d(xmm1);
4403 masm->push_d(xmm2);
4404 masm->push_d(xmm3);
4405 masm->push_d(xmm4);
4406 masm->push_d(xmm5);
4407 masm->push_d(xmm6);
4408 masm->push_d(xmm7);
4409 }
4410 #ifdef ASSERT
4411 masm->movq(rax, 0xBADC0FFE);
4412 masm->movq(rdi, rax);
4413 masm->movq(rsi, rax);
4414 masm->movq(rdx, rax);
4415 masm->movq(rcx, rax);
4416 masm->movq(r8, rax);
4417 masm->movq(r9, rax);
4418 masm->movq(xmm0, rax);
4419 masm->movq(xmm1, rax);
4420 masm->movq(xmm2, rax);
4421 masm->movq(xmm3, rax);
4422 masm->movq(xmm4, rax);
4423 masm->movq(xmm5, rax);
4424 masm->movq(xmm6, rax);
4425 masm->movq(xmm7, rax);
4426 #endif
4427 }
4428
4429 static void restore_return_registers(MacroAssembler* masm) {
4430 if (InlineTypeReturnedAsFields) {
4431 masm->pop_d(xmm7);
4432 masm->pop_d(xmm6);
4433 masm->pop_d(xmm5);
4434 masm->pop_d(xmm4);
4435 masm->pop_d(xmm3);
4436 masm->pop_d(xmm2);
4437 masm->pop_d(xmm1);
4438 }
4439 masm->pop_d(xmm0);
4440 if (InlineTypeReturnedAsFields) {
4441 masm->pop(r9);
4442 masm->pop(r8);
4443 masm->pop(rcx);
4444 masm->pop(rdx);
4445 masm->pop(rsi);
4446 masm->pop(rdi);
4447 }
4448 masm->pop_ppx(rax);
4449 }
4450
4451 address StubGenerator::generate_cont_thaw(StubId stub_id) {
4452 if (!Continuations::enabled()) return nullptr;
4453
4454 bool return_barrier;
4455 bool return_barrier_exception;
4456 Continuation::thaw_kind kind;
4457
4458 switch (stub_id) {
4459 case StubId::stubgen_cont_thaw_id:
4460 return_barrier = false;
4461 return_barrier_exception = false;
4462 kind = Continuation::thaw_top;
4463 break;
4464 case StubId::stubgen_cont_returnBarrier_id:
4465 return_barrier = true;
4466 return_barrier_exception = false;
4467 kind = Continuation::thaw_return_barrier;
4468 break;
4469 case StubId::stubgen_cont_returnBarrierExc_id:
4470 return_barrier = true;
4488 if (!return_barrier) {
4489 // Pop return address. If we don't do this, we get a drift,
4490 // where the bottom-most frozen frame continuously grows.
4491 __ pop(c_rarg3);
4492 } else {
4493 __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
4494 }
4495
4496 #ifdef ASSERT
4497 {
4498 Label L_good_sp;
4499 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
4500 __ jcc(Assembler::equal, L_good_sp);
4501 __ stop("Incorrect rsp at thaw entry");
4502 __ BIND(L_good_sp);
4503 }
4504 #endif // ASSERT
4505
4506 if (return_barrier) {
4507 // Preserve possible return value from a method returning to the return barrier.
4508 save_return_registers(_masm);
4509 }
4510
4511 __ movptr(c_rarg0, r15_thread);
4512 __ movptr(c_rarg1, (return_barrier ? 1 : 0));
4513 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), 2);
4514 __ movptr(rbx, rax);
4515
4516 if (return_barrier) {
4517 // Restore return value from a method returning to the return barrier.
4518 // No safepoint in the call to thaw, so even an oop return value should be OK.
4519 restore_return_registers(_masm);
4520 }
4521
4522 #ifdef ASSERT
4523 {
4524 Label L_good_sp;
4525 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
4526 __ jcc(Assembler::equal, L_good_sp);
4527 __ stop("Incorrect rsp after prepare thaw");
4528 __ BIND(L_good_sp);
4529 }
4530 #endif // ASSERT
4531
4532 // rbx contains the size of the frames to thaw, 0 if overflow or no more frames
4533 Label L_thaw_success;
4534 __ testptr(rbx, rbx);
4535 __ jccb(Assembler::notZero, L_thaw_success);
4536 __ jump(RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
4537 __ bind(L_thaw_success);
4538
4539 // Make room for the thawed frames and align the stack.
4540 __ subptr(rsp, rbx);
4541 __ andptr(rsp, -StackAlignmentInBytes);
4542
4543 if (return_barrier) {
4544 // Preserve possible return value from a method returning to the return barrier. (Again.)
4545 save_return_registers(_masm);
4546 }
4547
4548 // If we want, we can templatize thaw by kind, and have three different entries.
4549 __ movptr(c_rarg0, r15_thread);
4550 __ movptr(c_rarg1, kind);
4551 __ call_VM_leaf(Continuation::thaw_entry(), 2);
4552 __ movptr(rbx, rax);
4553
4554 if (return_barrier) {
4555 // Restore return value from a method returning to the return barrier. (Again.)
4556 // No safepoint in the call to thaw, so even an oop return value should be OK.
4557 restore_return_registers(_masm);
4558 } else {
4559 // Return 0 (success) from doYield.
4560 __ xorptr(rax, rax);
4561 }
4562
4563 // After thawing, rbx is the SP of the yielding frame.
4564 // Move there, and then to saved RBP slot.
4565 __ movptr(rsp, rbx);
4566 __ subptr(rsp, 2*wordSize);
4567
4568 if (return_barrier_exception) {
4569 __ movptr(c_rarg0, r15_thread);
4570 __ movptr(c_rarg1, Address(rsp, wordSize)); // return address
4571
4572 // rax still holds the original exception oop, save it before the call
4573 __ push_ppx(rax);
4574
4575 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), 2);
4576 __ movptr(rbx, rax);
4577
4804
4805 void StubGenerator::generate_initial_stubs() {
4806 // Generates all stubs and initializes the entry points
4807
4808 // This platform-specific settings are needed by generate_call_stub()
4809 create_control_words();
4810
4811 // Initialize table for unsafe copy memeory check.
4812 if (UnsafeMemoryAccess::_table == nullptr) {
4813 UnsafeMemoryAccess::create_table(16 + 4); // 16 for copyMemory; 4 for setMemory
4814 }
4815
4816 // entry points that exist in all platforms Note: This is code
4817 // that could be shared among different platforms - however the
4818 // benefit seems to be smaller than the disadvantage of having a
4819 // much more complicated generator structure. See also comment in
4820 // stubRoutines.hpp.
4821
4822 StubRoutines::_forward_exception_entry = generate_forward_exception();
4823
4824 // Generate these first because they are called from other stubs
4825 if (InlineTypeReturnedAsFields) {
4826 StubRoutines::_load_inline_type_fields_in_regs =
4827 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::load_inline_type_fields_in_regs),
4828 "load_inline_type_fields_in_regs", false);
4829 StubRoutines::_store_inline_type_fields_to_buf =
4830 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::store_inline_type_fields_to_buf),
4831 "store_inline_type_fields_to_buf", true);
4832 }
4833
4834 StubRoutines::_call_stub_entry =
4835 generate_call_stub(StubRoutines::_call_stub_return_address);
4836
4837 // is referenced by megamorphic call
4838 StubRoutines::_catch_exception_entry = generate_catch_exception();
4839
4840 // platform dependent
4841 StubRoutines::x86::_verify_mxcsr_entry = generate_verify_mxcsr();
4842
4843 StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
4844 StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
4845 StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
4846 StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
4847
4848 StubRoutines::x86::_float_sign_mask = generate_fp_mask(StubId::stubgen_float_sign_mask_id, 0x7FFFFFFF7FFFFFFF);
4849 StubRoutines::x86::_float_sign_flip = generate_fp_mask(StubId::stubgen_float_sign_flip_id, 0x8000000080000000);
4850 StubRoutines::x86::_double_sign_mask = generate_fp_mask(StubId::stubgen_double_sign_mask_id, 0x7FFFFFFFFFFFFFFF);
4851 StubRoutines::x86::_double_sign_flip = generate_fp_mask(StubId::stubgen_double_sign_flip_id, 0x8000000000000000);
4852
4853 if (UseCRC32Intrinsics) {
4857 if (UseCRC32CIntrinsics) {
4858 bool supports_clmul = VM_Version::supports_clmul();
4859 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
4860 }
4861
4862 if (VM_Version::supports_float16()) {
4863 // For results consistency both intrinsics should be enabled.
4864 // vmIntrinsics checks InlineIntrinsics flag, no need to check it here.
4865 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
4866 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
4867 StubRoutines::_hf2f = generate_float16ToFloat();
4868 StubRoutines::_f2hf = generate_floatToFloat16();
4869 }
4870 }
4871
4872 generate_libm_stubs();
4873
4874 StubRoutines::_fmod = generate_libmFmod(); // from stubGenerator_x86_64_fmod.cpp
4875 }
4876
4877 // Call here from the interpreter or compiled code to either load
4878 // multiple returned values from the inline type instance being
4879 // returned to registers or to store returned values to a newly
4880 // allocated inline type instance.
4881 // Register is a class, but it would be assigned numerical value.
4882 // "0" is assigned for xmm0. Thus we need to ignore -Wnonnull.
4883 PRAGMA_DIAG_PUSH
4884 PRAGMA_NONNULL_IGNORED
4885 address StubGenerator::generate_return_value_stub(address destination, const char* name, bool has_res) {
4886 // We need to save all registers the calling convention may use so
4887 // the runtime calls read or update those registers. This needs to
4888 // be in sync with SharedRuntime::java_return_convention().
4889 enum layout {
4890 pad_off = frame::arg_reg_save_area_bytes/BytesPerInt, pad_off_2,
4891 rax_off, rax_off_2,
4892 j_rarg5_off, j_rarg5_2,
4893 j_rarg4_off, j_rarg4_2,
4894 j_rarg3_off, j_rarg3_2,
4895 j_rarg2_off, j_rarg2_2,
4896 j_rarg1_off, j_rarg1_2,
4897 j_rarg0_off, j_rarg0_2,
4898 j_farg0_off, j_farg0_2,
4899 j_farg1_off, j_farg1_2,
4900 j_farg2_off, j_farg2_2,
4901 j_farg3_off, j_farg3_2,
4902 j_farg4_off, j_farg4_2,
4903 j_farg5_off, j_farg5_2,
4904 j_farg6_off, j_farg6_2,
4905 j_farg7_off, j_farg7_2,
4906 rbp_off, rbp_off_2,
4907 return_off, return_off_2,
4908
4909 framesize
4910 };
4911
4912 CodeBuffer buffer(name, 1000, 512);
4913 MacroAssembler* _masm = new MacroAssembler(&buffer);
4914
4915 int frame_size_in_bytes = align_up(framesize*BytesPerInt, 16);
4916 assert(frame_size_in_bytes == framesize*BytesPerInt, "misaligned");
4917 int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
4918 int frame_size_in_words = frame_size_in_bytes / wordSize;
4919
4920 OopMapSet *oop_maps = new OopMapSet();
4921 OopMap* map = new OopMap(frame_size_in_slots, 0);
4922
4923 map->set_callee_saved(VMRegImpl::stack2reg(rax_off), rax->as_VMReg());
4924 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg5_off), j_rarg5->as_VMReg());
4925 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg4_off), j_rarg4->as_VMReg());
4926 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg3_off), j_rarg3->as_VMReg());
4927 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg2_off), j_rarg2->as_VMReg());
4928 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg1_off), j_rarg1->as_VMReg());
4929 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg0_off), j_rarg0->as_VMReg());
4930 map->set_callee_saved(VMRegImpl::stack2reg(j_farg0_off), j_farg0->as_VMReg());
4931 map->set_callee_saved(VMRegImpl::stack2reg(j_farg1_off), j_farg1->as_VMReg());
4932 map->set_callee_saved(VMRegImpl::stack2reg(j_farg2_off), j_farg2->as_VMReg());
4933 map->set_callee_saved(VMRegImpl::stack2reg(j_farg3_off), j_farg3->as_VMReg());
4934 map->set_callee_saved(VMRegImpl::stack2reg(j_farg4_off), j_farg4->as_VMReg());
4935 map->set_callee_saved(VMRegImpl::stack2reg(j_farg5_off), j_farg5->as_VMReg());
4936 map->set_callee_saved(VMRegImpl::stack2reg(j_farg6_off), j_farg6->as_VMReg());
4937 map->set_callee_saved(VMRegImpl::stack2reg(j_farg7_off), j_farg7->as_VMReg());
4938
4939 int start = __ offset();
4940
4941 __ subptr(rsp, frame_size_in_bytes - 8 /* return address*/);
4942
4943 __ movptr(Address(rsp, rbp_off * BytesPerInt), rbp);
4944 __ movdbl(Address(rsp, j_farg7_off * BytesPerInt), j_farg7);
4945 __ movdbl(Address(rsp, j_farg6_off * BytesPerInt), j_farg6);
4946 __ movdbl(Address(rsp, j_farg5_off * BytesPerInt), j_farg5);
4947 __ movdbl(Address(rsp, j_farg4_off * BytesPerInt), j_farg4);
4948 __ movdbl(Address(rsp, j_farg3_off * BytesPerInt), j_farg3);
4949 __ movdbl(Address(rsp, j_farg2_off * BytesPerInt), j_farg2);
4950 __ movdbl(Address(rsp, j_farg1_off * BytesPerInt), j_farg1);
4951 __ movdbl(Address(rsp, j_farg0_off * BytesPerInt), j_farg0);
4952
4953 __ movptr(Address(rsp, j_rarg0_off * BytesPerInt), j_rarg0);
4954 __ movptr(Address(rsp, j_rarg1_off * BytesPerInt), j_rarg1);
4955 __ movptr(Address(rsp, j_rarg2_off * BytesPerInt), j_rarg2);
4956 __ movptr(Address(rsp, j_rarg3_off * BytesPerInt), j_rarg3);
4957 __ movptr(Address(rsp, j_rarg4_off * BytesPerInt), j_rarg4);
4958 __ movptr(Address(rsp, j_rarg5_off * BytesPerInt), j_rarg5);
4959 __ movptr(Address(rsp, rax_off * BytesPerInt), rax);
4960
4961 int frame_complete = __ offset();
4962
4963 __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1);
4964
4965 __ mov(c_rarg0, r15_thread);
4966 __ mov(c_rarg1, rax);
4967
4968 __ call(RuntimeAddress(destination));
4969
4970 // Set an oopmap for the call site.
4971
4972 oop_maps->add_gc_map( __ offset() - start, map);
4973
4974 // clear last_Java_sp
4975 __ reset_last_Java_frame(false);
4976
4977 __ movptr(rbp, Address(rsp, rbp_off * BytesPerInt));
4978 __ movdbl(j_farg7, Address(rsp, j_farg7_off * BytesPerInt));
4979 __ movdbl(j_farg6, Address(rsp, j_farg6_off * BytesPerInt));
4980 __ movdbl(j_farg5, Address(rsp, j_farg5_off * BytesPerInt));
4981 __ movdbl(j_farg4, Address(rsp, j_farg4_off * BytesPerInt));
4982 __ movdbl(j_farg3, Address(rsp, j_farg3_off * BytesPerInt));
4983 __ movdbl(j_farg2, Address(rsp, j_farg2_off * BytesPerInt));
4984 __ movdbl(j_farg1, Address(rsp, j_farg1_off * BytesPerInt));
4985 __ movdbl(j_farg0, Address(rsp, j_farg0_off * BytesPerInt));
4986
4987 __ movptr(j_rarg0, Address(rsp, j_rarg0_off * BytesPerInt));
4988 __ movptr(j_rarg1, Address(rsp, j_rarg1_off * BytesPerInt));
4989 __ movptr(j_rarg2, Address(rsp, j_rarg2_off * BytesPerInt));
4990 __ movptr(j_rarg3, Address(rsp, j_rarg3_off * BytesPerInt));
4991 __ movptr(j_rarg4, Address(rsp, j_rarg4_off * BytesPerInt));
4992 __ movptr(j_rarg5, Address(rsp, j_rarg5_off * BytesPerInt));
4993 __ movptr(rax, Address(rsp, rax_off * BytesPerInt));
4994
4995 __ addptr(rsp, frame_size_in_bytes-8);
4996
4997 // check for pending exceptions
4998 Label pending;
4999 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
5000 __ jcc(Assembler::notEqual, pending);
5001
5002 if (has_res) {
5003 // We just called SharedRuntime::store_inline_type_fields_to_buf. Check if we still
5004 // need to initialize the buffer and if so, call the inline class specific pack handler.
5005 Label skip_pack;
5006 __ get_vm_result_oop(rax);
5007 __ get_vm_result_metadata(rscratch1);
5008 __ testptr(rscratch1, rscratch1);
5009 __ jcc(Assembler::zero, skip_pack);
5010 __ movptr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
5011 __ movptr(rscratch1, Address(rscratch1, InlineKlass::pack_handler_offset()));
5012 __ call(rscratch1);
5013 __ membar(Assembler::StoreStore);
5014 __ bind(skip_pack);
5015 }
5016
5017 __ ret(0);
5018
5019 __ bind(pending);
5020
5021 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
5022 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
5023
5024 // -------------
5025 // make sure all code is generated
5026 _masm->flush();
5027
5028 RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, false);
5029 return stub->entry_point();
5030 }
5031
5032 void StubGenerator::generate_continuation_stubs() {
5033 // Continuation stubs:
5034 StubRoutines::_cont_thaw = generate_cont_thaw();
5035 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
5036 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
5037 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
5038 }
5039
5040 void StubGenerator::generate_final_stubs() {
5041 // Generates the rest of stubs and initializes the entry points
5042
5043 // support for verify_oop (must happen after universe_init)
5044 if (VerifyOops) {
5045 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
5046 }
5047
5048 // arraycopy stubs used by compilers
5049 generate_arraycopy_stubs();
5050
5051 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
|