487 __ blr(c_rarg4);
488
489 // we do this here because the notify will already have been done
490 // if we get to the next instruction via an exception
491 //
492 // n.b. adding this instruction here affects the calculation of
493 // whether or not a routine returns to the call stub (used when
494 // doing stack walks) since the normal test is to check the return
495 // pc against the address saved below. so we may need to allow for
496 // this extra instruction in the check.
497
498 // save current address for use by exception handling code
499
500 return_address = __ pc();
501 entries.append(return_address);
502
503 // store result depending on type (everything that is not
504 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
505 // n.b. this assumes Java returns an integral result in r0
506 // and a floating result in j_farg0
507 __ ldr(j_rarg2, result);
508 Label is_long, is_float, is_double, exit;
509 __ ldr(j_rarg1, result_type);
510 __ cmp(j_rarg1, (u1)T_OBJECT);
511 __ br(Assembler::EQ, is_long);
512 __ cmp(j_rarg1, (u1)T_LONG);
513 __ br(Assembler::EQ, is_long);
514 __ cmp(j_rarg1, (u1)T_FLOAT);
515 __ br(Assembler::EQ, is_float);
516 __ cmp(j_rarg1, (u1)T_DOUBLE);
517 __ br(Assembler::EQ, is_double);
518
519 // handle T_INT case
520 __ strw(r0, Address(j_rarg2));
521
522 __ BIND(exit);
523
524 // pop parameters
525 __ sub(esp, rfp, -sp_after_call_off * wordSize);
526
527 #ifdef ASSERT
528 // verify that threads correspond
529 {
530 Label L, S;
531 __ ldr(rscratch1, thread);
532 __ cmp(rthread, rscratch1);
533 __ br(Assembler::NE, S);
534 __ get_thread(rscratch1);
535 __ cmp(rthread, rscratch1);
536 __ br(Assembler::EQ, L);
537 __ BIND(S);
538 __ stop("StubRoutines::call_stub: threads must correspond");
539 __ BIND(L);
540 }
552 __ ldp(r26, r25, r26_save);
553 __ ldp(r24, r23, r24_save);
554 __ ldp(r22, r21, r22_save);
555 __ ldp(r20, r19, r20_save);
556
557 // restore fpcr
558 __ ldr(rscratch1, fpcr_save);
559 __ set_fpcr(rscratch1);
560
561 __ ldp(c_rarg0, c_rarg1, call_wrapper);
562 __ ldrw(c_rarg2, result_type);
563 __ ldr(c_rarg3, method);
564 __ ldp(c_rarg4, c_rarg5, entry_point);
565 __ ldp(c_rarg6, c_rarg7, parameter_size);
566
567 // leave frame and return to caller
568 __ leave();
569 __ ret(lr);
570
571 // handle return types different from T_INT
572
573 __ BIND(is_long);
574 __ str(r0, Address(j_rarg2, 0));
575 __ br(Assembler::AL, exit);
576
577 __ BIND(is_float);
578 __ strs(j_farg0, Address(j_rarg2, 0));
579 __ br(Assembler::AL, exit);
580
581 __ BIND(is_double);
582 __ strd(j_farg0, Address(j_rarg2, 0));
583 __ br(Assembler::AL, exit);
584
585 // record the stub entry and end plus the auxiliary entry
586 store_archive_data(stub_id, start, __ pc(), &entries);
587
588 return start;
589 }
590
591 // Return point for a Java call if there's an exception thrown in
592 // Java code. The exception is caught and transformed into a
593 // pending exception stored in JavaThread that can be tested from
594 // within the VM.
595 //
596 // Note: Usually the parameters are removed by the callee. In case
597 // of an exception crossing an activation frame boundary, that is
598 // not the case if the callee is compiled code => need to setup the
599 // rsp.
600 //
601 // r0: exception oop
602
2589 // |array_tag| | header_size | element_type | |log2_element_size|
2590 // 32 30 24 16 8 2 0
2591 //
2592 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2593 //
2594
2595 const int lh_offset = in_bytes(Klass::layout_helper_offset());
2596
2597 // Handle objArrays completely differently...
2598 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2599 __ ldrw(lh, Address(scratch_src_klass, lh_offset));
2600 __ movw(rscratch1, objArray_lh);
2601 __ eorw(rscratch2, lh, rscratch1);
2602 __ cbzw(rscratch2, L_objArray);
2603
2604 // if (src->klass() != dst->klass()) return -1;
2605 __ load_klass(rscratch2, dst);
2606 __ eor(rscratch2, rscratch2, scratch_src_klass);
2607 __ cbnz(rscratch2, L_failed);
2608
2609 // if (!src->is_Array()) return -1;
2610 __ tbz(lh, 31, L_failed); // i.e. (lh >= 0)
2611
2612 // At this point, it is known to be a typeArray (array_tag 0x3).
2613 #ifdef ASSERT
2614 {
2615 BLOCK_COMMENT("assert primitive array {");
2616 Label L;
2617 __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2618 __ cmpw(lh, rscratch2);
2619 __ br(Assembler::GE, L);
2620 __ stop("must be a primitive array");
2621 __ bind(L);
2622 BLOCK_COMMENT("} assert primitive array done");
2623 }
2624 #endif
2625
2626 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2627 rscratch2, L_failed);
2628
12380 entries.append((address)aarch64_atomic_xchg_4_impl);
12381 entries.append((address)aarch64_atomic_xchg_8_impl);
12382 entries.append((address)aarch64_atomic_cmpxchg_1_impl);
12383 entries.append((address)aarch64_atomic_cmpxchg_4_impl);
12384 entries.append((address)aarch64_atomic_cmpxchg_8_impl);
12385 entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl);
12386 entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl);
12387 entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl);
12388 entries.append((address)aarch64_atomic_cmpxchg_4_release_impl);
12389 entries.append((address)aarch64_atomic_cmpxchg_8_release_impl);
12390 entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl);
12391 entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl);
12392
12393 assert(entries.length() == entry_count - 1,
12394 "unexpected extra entry count %d", entries.length());
12395
12396 store_archive_data(stub_id, start, end, &entries);
12397 }
12398 #endif // LINUX
12399
12400 address generate_cont_thaw(Continuation::thaw_kind kind) {
12401 bool return_barrier = Continuation::is_thaw_return_barrier(kind);
12402 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
12403
12404 address start = __ pc();
12405
12406 if (return_barrier) {
12407 __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
12408 __ mov(sp, rscratch1);
12409 }
12410 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
12411
12412 if (return_barrier) {
12413 // preserve possible return value from a method returning to the return barrier
12414 __ fmovd(rscratch1, v0);
12415 __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
12416 }
12417
12418 __ movw(c_rarg1, (return_barrier ? 1 : 0));
12419 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
12420 __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
12421
12422 if (return_barrier) {
12423 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
12424 __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
12425 __ fmovd(v0, rscratch1);
12426 }
12427 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
12428
12429
12430 Label thaw_success;
12431 // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
12432 __ cbnz(rscratch2, thaw_success);
12433 __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
12434 __ br(rscratch1);
12435 __ bind(thaw_success);
12436
12437 // make room for the thawed frames
12438 __ sub(rscratch1, sp, rscratch2);
12439 __ andr(rscratch1, rscratch1, -16); // align
12440 __ mov(sp, rscratch1);
12441
12442 if (return_barrier) {
12443 // save original return value -- again
12444 __ fmovd(rscratch1, v0);
12445 __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
12446 }
12447
12448 // If we want, we can templatize thaw by kind, and have three different entries
12449 __ movw(c_rarg1, (uint32_t)kind);
12450
12451 __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
12452 __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
12453
12454 if (return_barrier) {
12455 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
12456 __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
12457 __ fmovd(v0, rscratch1);
12458 } else {
12459 __ mov(r0, zr); // return 0 (success) from doYield
12460 }
12461
12462 // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
12463 __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
12464 __ mov(rfp, sp);
12465
12466 if (return_barrier_exception) {
12467 __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
12468 __ authenticate_return_address(c_rarg1);
12469 __ verify_oop(r0);
12470 // save return value containing the exception oop in callee-saved R19
12471 __ mov(r19, r0);
12472
12473 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
12474
12475 // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
12476 // __ reinitialize_ptrue();
12477
13642 // assert(Ra == Pa_base[j], "must be");
13643 // MACC(Ra, Ra, t0, t1, t2);
13644 // }
13645 // iters = (2*len-i)/2;
13646 // assert(iters == len-j, "must be");
13647 // for (; iters--; j++) {
13648 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
13649 // MACC(Rm, Rn, t0, t1, t2);
13650 // Rm = *++Pm;
13651 // Rn = *--Pn;
13652 // }
13653 // Pm_base[i-len] = t0;
13654 // t0 = t1; t1 = t2; t2 = 0;
13655 // }
13656
13657 // while (t0)
13658 // t0 = sub(Pm_base, Pn_base, t0, len);
13659 // }
13660 };
13661
13662 // Initialization
13663 void generate_preuniverse_stubs() {
13664 // preuniverse stubs are not needed for aarch64
13665 }
13666
13667 void generate_initial_stubs() {
13668 // Generate initial stubs and initializes the entry points
13669
13670 // entry points that exist in all platforms Note: This is code
13671 // that could be shared among different platforms - however the
13672 // benefit seems to be smaller than the disadvantage of having a
13673 // much more complicated generator structure. See also comment in
13674 // stubRoutines.hpp.
13675
13676 StubRoutines::_forward_exception_entry = generate_forward_exception();
13677
13678 StubRoutines::_call_stub_entry =
13679 generate_call_stub(StubRoutines::_call_stub_return_address);
13680
13681 // is referenced by megamorphic call
13690 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
13691 }
13692
13693 if (UseCRC32CIntrinsics) {
13694 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
13695 }
13696
13697 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
13698 StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
13699 }
13700
13701 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
13702 StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
13703 }
13704
13705 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
13706 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
13707 StubRoutines::_hf2f = generate_float16ToFloat();
13708 StubRoutines::_f2hf = generate_floatToFloat16();
13709 }
13710 }
13711
13712 void generate_continuation_stubs() {
13713 // Continuation stubs:
13714 StubRoutines::_cont_thaw = generate_cont_thaw();
13715 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
13716 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
13717 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
13718 }
13719
13720 void generate_final_stubs() {
13721 // support for verify_oop (must happen after universe_init)
13722 if (VerifyOops) {
13723 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
13724 }
13725
13726 // arraycopy stubs used by compilers
13727 generate_arraycopy_stubs();
13728
13729 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
|
487 __ blr(c_rarg4);
488
489 // we do this here because the notify will already have been done
490 // if we get to the next instruction via an exception
491 //
492 // n.b. adding this instruction here affects the calculation of
493 // whether or not a routine returns to the call stub (used when
494 // doing stack walks) since the normal test is to check the return
495 // pc against the address saved below. so we may need to allow for
496 // this extra instruction in the check.
497
498 // save current address for use by exception handling code
499
500 return_address = __ pc();
501 entries.append(return_address);
502
503 // store result depending on type (everything that is not
504 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
505 // n.b. this assumes Java returns an integral result in r0
506 // and a floating result in j_farg0
507 // All of j_rargN may be used to return inline type fields so be careful
508 // not to clobber those.
509 // SharedRuntime::generate_buffered_inline_type_adapter() knows the register
510 // assignment of Rresult below.
511 Register Rresult = r14, Rresult_type = r15;
512 __ ldr(Rresult, result);
513 Label is_long, is_float, is_double, check_prim, exit;
514 __ ldr(Rresult_type, result_type);
515 __ cmp(Rresult_type, (u1)T_OBJECT);
516 __ br(Assembler::EQ, check_prim);
517 __ cmp(Rresult_type, (u1)T_LONG);
518 __ br(Assembler::EQ, is_long);
519 __ cmp(Rresult_type, (u1)T_FLOAT);
520 __ br(Assembler::EQ, is_float);
521 __ cmp(Rresult_type, (u1)T_DOUBLE);
522 __ br(Assembler::EQ, is_double);
523
524 // handle T_INT case
525 __ strw(r0, Address(Rresult));
526
527 __ BIND(exit);
528
529 // pop parameters
530 __ sub(esp, rfp, -sp_after_call_off * wordSize);
531
532 #ifdef ASSERT
533 // verify that threads correspond
534 {
535 Label L, S;
536 __ ldr(rscratch1, thread);
537 __ cmp(rthread, rscratch1);
538 __ br(Assembler::NE, S);
539 __ get_thread(rscratch1);
540 __ cmp(rthread, rscratch1);
541 __ br(Assembler::EQ, L);
542 __ BIND(S);
543 __ stop("StubRoutines::call_stub: threads must correspond");
544 __ BIND(L);
545 }
557 __ ldp(r26, r25, r26_save);
558 __ ldp(r24, r23, r24_save);
559 __ ldp(r22, r21, r22_save);
560 __ ldp(r20, r19, r20_save);
561
562 // restore fpcr
563 __ ldr(rscratch1, fpcr_save);
564 __ set_fpcr(rscratch1);
565
566 __ ldp(c_rarg0, c_rarg1, call_wrapper);
567 __ ldrw(c_rarg2, result_type);
568 __ ldr(c_rarg3, method);
569 __ ldp(c_rarg4, c_rarg5, entry_point);
570 __ ldp(c_rarg6, c_rarg7, parameter_size);
571
572 // leave frame and return to caller
573 __ leave();
574 __ ret(lr);
575
576 // handle return types different from T_INT
577 __ BIND(check_prim);
578 if (InlineTypeReturnedAsFields) {
579 // Check for scalarized return value
580 __ tbz(r0, 0, is_long);
581 // Load pack handler address
582 __ andr(rscratch1, r0, -2);
583 __ ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
584 __ ldr(rscratch1, Address(rscratch1, InlineKlass::pack_handler_jobject_offset()));
585 __ blr(rscratch1);
586 __ b(exit);
587 }
588
589 __ BIND(is_long);
590 __ str(r0, Address(Rresult, 0));
591 __ br(Assembler::AL, exit);
592
593 __ BIND(is_float);
594 __ strs(j_farg0, Address(Rresult, 0));
595 __ br(Assembler::AL, exit);
596
597 __ BIND(is_double);
598 __ strd(j_farg0, Address(Rresult, 0));
599 __ br(Assembler::AL, exit);
600
601 // record the stub entry and end plus the auxiliary entry
602 store_archive_data(stub_id, start, __ pc(), &entries);
603
604 return start;
605 }
606
607 // Return point for a Java call if there's an exception thrown in
608 // Java code. The exception is caught and transformed into a
609 // pending exception stored in JavaThread that can be tested from
610 // within the VM.
611 //
612 // Note: Usually the parameters are removed by the callee. In case
613 // of an exception crossing an activation frame boundary, that is
614 // not the case if the callee is compiled code => need to setup the
615 // rsp.
616 //
617 // r0: exception oop
618
2605 // |array_tag| | header_size | element_type | |log2_element_size|
2606 // 32 30 24 16 8 2 0
2607 //
2608 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2609 //
2610
2611 const int lh_offset = in_bytes(Klass::layout_helper_offset());
2612
2613 // Handle objArrays completely differently...
2614 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2615 __ ldrw(lh, Address(scratch_src_klass, lh_offset));
2616 __ movw(rscratch1, objArray_lh);
2617 __ eorw(rscratch2, lh, rscratch1);
2618 __ cbzw(rscratch2, L_objArray);
2619
2620 // if (src->klass() != dst->klass()) return -1;
2621 __ load_klass(rscratch2, dst);
2622 __ eor(rscratch2, rscratch2, scratch_src_klass);
2623 __ cbnz(rscratch2, L_failed);
2624
2625 // Check for flat inline type array -> return -1
2626 __ test_flat_array_oop(src, rscratch2, L_failed);
2627
2628 // Check for null-free (non-flat) inline type array -> handle as object array
2629 __ test_null_free_array_oop(src, rscratch2, L_objArray);
2630
2631 // if (!src->is_Array()) return -1;
2632 __ tbz(lh, 31, L_failed); // i.e. (lh >= 0)
2633
2634 // At this point, it is known to be a typeArray (array_tag 0x3).
2635 #ifdef ASSERT
2636 {
2637 BLOCK_COMMENT("assert primitive array {");
2638 Label L;
2639 __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2640 __ cmpw(lh, rscratch2);
2641 __ br(Assembler::GE, L);
2642 __ stop("must be a primitive array");
2643 __ bind(L);
2644 BLOCK_COMMENT("} assert primitive array done");
2645 }
2646 #endif
2647
2648 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2649 rscratch2, L_failed);
2650
12402 entries.append((address)aarch64_atomic_xchg_4_impl);
12403 entries.append((address)aarch64_atomic_xchg_8_impl);
12404 entries.append((address)aarch64_atomic_cmpxchg_1_impl);
12405 entries.append((address)aarch64_atomic_cmpxchg_4_impl);
12406 entries.append((address)aarch64_atomic_cmpxchg_8_impl);
12407 entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl);
12408 entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl);
12409 entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl);
12410 entries.append((address)aarch64_atomic_cmpxchg_4_release_impl);
12411 entries.append((address)aarch64_atomic_cmpxchg_8_release_impl);
12412 entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl);
12413 entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl);
12414
12415 assert(entries.length() == entry_count - 1,
12416 "unexpected extra entry count %d", entries.length());
12417
12418 store_archive_data(stub_id, start, end, &entries);
12419 }
12420 #endif // LINUX
12421
12422 static void save_return_registers(MacroAssembler* masm) {
12423 if (InlineTypeReturnedAsFields) {
12424 masm->push(RegSet::range(r0, r7), sp);
12425 masm->sub(sp, sp, 4 * wordSize);
12426 masm->st1(v0, v1, v2, v3, masm->T1D, Address(sp));
12427 masm->sub(sp, sp, 4 * wordSize);
12428 masm->st1(v4, v5, v6, v7, masm->T1D, Address(sp));
12429 } else {
12430 masm->fmovd(rscratch1, v0);
12431 masm->stp(rscratch1, r0, Address(masm->pre(sp, -2 * wordSize)));
12432 }
12433 }
12434
12435 static void restore_return_registers(MacroAssembler* masm) {
12436 if (InlineTypeReturnedAsFields) {
12437 masm->ld1(v4, v5, v6, v7, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
12438 masm->ld1(v0, v1, v2, v3, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
12439 masm->pop(RegSet::range(r0, r7), sp);
12440 } else {
12441 masm->ldp(rscratch1, r0, Address(masm->post(sp, 2 * wordSize)));
12442 masm->fmovd(v0, rscratch1);
12443 }
12444 }
12445
12446 address generate_cont_thaw(Continuation::thaw_kind kind) {
12447 bool return_barrier = Continuation::is_thaw_return_barrier(kind);
12448 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
12449
12450 address start = __ pc();
12451
12452 if (return_barrier) {
12453 __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
12454 __ mov(sp, rscratch1);
12455 }
12456 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
12457
12458 if (return_barrier) {
12459 // preserve possible return value from a method returning to the return barrier
12460 save_return_registers(_masm);
12461 }
12462
12463 __ movw(c_rarg1, (return_barrier ? 1 : 0));
12464 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
12465 __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
12466
12467 if (return_barrier) {
12468 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
12469 restore_return_registers(_masm);
12470 }
12471 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
12472
12473
12474 Label thaw_success;
12475 // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
12476 __ cbnz(rscratch2, thaw_success);
12477 __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
12478 __ br(rscratch1);
12479 __ bind(thaw_success);
12480
12481 // make room for the thawed frames
12482 __ sub(rscratch1, sp, rscratch2);
12483 __ andr(rscratch1, rscratch1, -16); // align
12484 __ mov(sp, rscratch1);
12485
12486 if (return_barrier) {
12487 // save original return value -- again
12488 save_return_registers(_masm);
12489 }
12490
12491 // If we want, we can templatize thaw by kind, and have three different entries
12492 __ movw(c_rarg1, (uint32_t)kind);
12493
12494 __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
12495 __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
12496
12497 if (return_barrier) {
12498 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
12499 restore_return_registers(_masm);
12500 } else {
12501 __ mov(r0, zr); // return 0 (success) from doYield
12502 }
12503
12504 // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
12505 __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
12506 __ mov(rfp, sp);
12507
12508 if (return_barrier_exception) {
12509 __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
12510 __ authenticate_return_address(c_rarg1);
12511 __ verify_oop(r0);
12512 // save return value containing the exception oop in callee-saved R19
12513 __ mov(r19, r0);
12514
12515 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
12516
12517 // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
12518 // __ reinitialize_ptrue();
12519
13684 // assert(Ra == Pa_base[j], "must be");
13685 // MACC(Ra, Ra, t0, t1, t2);
13686 // }
13687 // iters = (2*len-i)/2;
13688 // assert(iters == len-j, "must be");
13689 // for (; iters--; j++) {
13690 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
13691 // MACC(Rm, Rn, t0, t1, t2);
13692 // Rm = *++Pm;
13693 // Rn = *--Pn;
13694 // }
13695 // Pm_base[i-len] = t0;
13696 // t0 = t1; t1 = t2; t2 = 0;
13697 // }
13698
13699 // while (t0)
13700 // t0 = sub(Pm_base, Pn_base, t0, len);
13701 // }
13702 };
13703
13704 // Call here from the interpreter or compiled code to either load
13705 // multiple returned values from the inline type instance being
13706 // returned to registers or to store returned values to a newly
13707 // allocated inline type instance.
13708 address generate_return_value_stub(address destination, const char* name, bool has_res) {
13709 // We need to save all registers the calling convention may use so
13710 // the runtime calls read or update those registers. This needs to
13711 // be in sync with SharedRuntime::java_return_convention().
13712 // n.b. aarch64 asserts that frame::arg_reg_save_area_bytes == 0
13713 enum layout {
13714 j_rarg7_off = 0, j_rarg7_2, // j_rarg7 is r0
13715 j_rarg6_off, j_rarg6_2,
13716 j_rarg5_off, j_rarg5_2,
13717 j_rarg4_off, j_rarg4_2,
13718 j_rarg3_off, j_rarg3_2,
13719 j_rarg2_off, j_rarg2_2,
13720 j_rarg1_off, j_rarg1_2,
13721 j_rarg0_off, j_rarg0_2,
13722
13723 j_farg7_off, j_farg7_2,
13724 j_farg6_off, j_farg6_2,
13725 j_farg5_off, j_farg5_2,
13726 j_farg4_off, j_farg4_2,
13727 j_farg3_off, j_farg3_2,
13728 j_farg2_off, j_farg2_2,
13729 j_farg1_off, j_farg1_2,
13730 j_farg0_off, j_farg0_2,
13731
13732 rfp_off, rfp_off2,
13733 return_off, return_off2,
13734
13735 framesize // inclusive of return address
13736 };
13737
13738 CodeBuffer code(name, 512, 64);
13739 MacroAssembler* masm = new MacroAssembler(&code);
13740
13741 int frame_size_in_bytes = align_up(framesize*BytesPerInt, 16);
13742 assert(frame_size_in_bytes == framesize*BytesPerInt, "misaligned");
13743 int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
13744 int frame_size_in_words = frame_size_in_bytes / wordSize;
13745
13746 OopMapSet* oop_maps = new OopMapSet();
13747 OopMap* map = new OopMap(frame_size_in_slots, 0);
13748
13749 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg7_off), j_rarg7->as_VMReg());
13750 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg6_off), j_rarg6->as_VMReg());
13751 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg5_off), j_rarg5->as_VMReg());
13752 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg4_off), j_rarg4->as_VMReg());
13753 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg3_off), j_rarg3->as_VMReg());
13754 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg2_off), j_rarg2->as_VMReg());
13755 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg1_off), j_rarg1->as_VMReg());
13756 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg0_off), j_rarg0->as_VMReg());
13757
13758 map->set_callee_saved(VMRegImpl::stack2reg(j_farg0_off), j_farg0->as_VMReg());
13759 map->set_callee_saved(VMRegImpl::stack2reg(j_farg1_off), j_farg1->as_VMReg());
13760 map->set_callee_saved(VMRegImpl::stack2reg(j_farg2_off), j_farg2->as_VMReg());
13761 map->set_callee_saved(VMRegImpl::stack2reg(j_farg3_off), j_farg3->as_VMReg());
13762 map->set_callee_saved(VMRegImpl::stack2reg(j_farg4_off), j_farg4->as_VMReg());
13763 map->set_callee_saved(VMRegImpl::stack2reg(j_farg5_off), j_farg5->as_VMReg());
13764 map->set_callee_saved(VMRegImpl::stack2reg(j_farg6_off), j_farg6->as_VMReg());
13765 map->set_callee_saved(VMRegImpl::stack2reg(j_farg7_off), j_farg7->as_VMReg());
13766
13767 address start = __ pc();
13768
13769 __ enter(); // Save FP and LR before call
13770
13771 __ stpd(j_farg1, j_farg0, Address(__ pre(sp, -2 * wordSize)));
13772 __ stpd(j_farg3, j_farg2, Address(__ pre(sp, -2 * wordSize)));
13773 __ stpd(j_farg5, j_farg4, Address(__ pre(sp, -2 * wordSize)));
13774 __ stpd(j_farg7, j_farg6, Address(__ pre(sp, -2 * wordSize)));
13775
13776 __ stp(j_rarg1, j_rarg0, Address(__ pre(sp, -2 * wordSize)));
13777 __ stp(j_rarg3, j_rarg2, Address(__ pre(sp, -2 * wordSize)));
13778 __ stp(j_rarg5, j_rarg4, Address(__ pre(sp, -2 * wordSize)));
13779 __ stp(j_rarg7, j_rarg6, Address(__ pre(sp, -2 * wordSize)));
13780
13781 int frame_complete = __ offset();
13782
13783 // Set up last_Java_sp and last_Java_fp
13784 address the_pc = __ pc();
13785 __ set_last_Java_frame(sp, noreg, the_pc, rscratch1);
13786
13787 // Call runtime
13788 __ mov(c_rarg1, r0);
13789 __ mov(c_rarg0, rthread);
13790
13791 __ mov(rscratch1, destination);
13792 __ blr(rscratch1);
13793
13794 oop_maps->add_gc_map(the_pc - start, map);
13795
13796 __ reset_last_Java_frame(false);
13797
13798 __ ldp(j_rarg7, j_rarg6, Address(__ post(sp, 2 * wordSize)));
13799 __ ldp(j_rarg5, j_rarg4, Address(__ post(sp, 2 * wordSize)));
13800 __ ldp(j_rarg3, j_rarg2, Address(__ post(sp, 2 * wordSize)));
13801 __ ldp(j_rarg1, j_rarg0, Address(__ post(sp, 2 * wordSize)));
13802
13803 __ ldpd(j_farg7, j_farg6, Address(__ post(sp, 2 * wordSize)));
13804 __ ldpd(j_farg5, j_farg4, Address(__ post(sp, 2 * wordSize)));
13805 __ ldpd(j_farg3, j_farg2, Address(__ post(sp, 2 * wordSize)));
13806 __ ldpd(j_farg1, j_farg0, Address(__ post(sp, 2 * wordSize)));
13807
13808 // check for pending exceptions
13809 Label pending;
13810 __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
13811 __ cbnz(rscratch1, pending);
13812
13813 if (has_res) {
13814 // We just called SharedRuntime::store_inline_type_fields_to_buf. Check if we still
13815 // need to initialize the buffer and if so, call the inline class specific pack handler.
13816 Label skip_pack;
13817 __ get_vm_result_oop(r0, rthread);
13818 __ get_vm_result_metadata(rscratch1, rthread);
13819 __ cbz(rscratch1, skip_pack);
13820 __ ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
13821 __ ldr(rscratch1, Address(rscratch1, InlineKlass::pack_handler_offset()));
13822 __ blr(rscratch1);
13823 __ membar(Assembler::StoreStore);
13824 __ bind(skip_pack);
13825 }
13826
13827 __ leave();
13828 __ ret(lr);
13829
13830 __ bind(pending);
13831 __ leave();
13832 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
13833
13834 // -------------
13835 // make sure all code is generated
13836 masm->flush();
13837
13838 RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, frame_size_in_words, oop_maps, false);
13839 return stub->entry_point();
13840 }
13841
13842 // Initialization
13843 void generate_preuniverse_stubs() {
13844 // preuniverse stubs are not needed for aarch64
13845 }
13846
13847 void generate_initial_stubs() {
13848 // Generate initial stubs and initializes the entry points
13849
13850 // entry points that exist in all platforms Note: This is code
13851 // that could be shared among different platforms - however the
13852 // benefit seems to be smaller than the disadvantage of having a
13853 // much more complicated generator structure. See also comment in
13854 // stubRoutines.hpp.
13855
13856 StubRoutines::_forward_exception_entry = generate_forward_exception();
13857
13858 StubRoutines::_call_stub_entry =
13859 generate_call_stub(StubRoutines::_call_stub_return_address);
13860
13861 // is referenced by megamorphic call
13870 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
13871 }
13872
13873 if (UseCRC32CIntrinsics) {
13874 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
13875 }
13876
13877 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
13878 StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
13879 }
13880
13881 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
13882 StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
13883 }
13884
13885 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
13886 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
13887 StubRoutines::_hf2f = generate_float16ToFloat();
13888 StubRoutines::_f2hf = generate_floatToFloat16();
13889 }
13890
13891 if (InlineTypeReturnedAsFields) {
13892 StubRoutines::_load_inline_type_fields_in_regs =
13893 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::load_inline_type_fields_in_regs), "load_inline_type_fields_in_regs", false);
13894 StubRoutines::_store_inline_type_fields_to_buf =
13895 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::store_inline_type_fields_to_buf), "store_inline_type_fields_to_buf", true);
13896 }
13897
13898 }
13899
13900 void generate_continuation_stubs() {
13901 // Continuation stubs:
13902 StubRoutines::_cont_thaw = generate_cont_thaw();
13903 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
13904 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
13905 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
13906 }
13907
13908 void generate_final_stubs() {
13909 // support for verify_oop (must happen after universe_init)
13910 if (VerifyOops) {
13911 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
13912 }
13913
13914 // arraycopy stubs used by compilers
13915 generate_arraycopy_stubs();
13916
13917 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
|