481 __ blr(c_rarg4);
482
483 // we do this here because the notify will already have been done
484 // if we get to the next instruction via an exception
485 //
486 // n.b. adding this instruction here affects the calculation of
487 // whether or not a routine returns to the call stub (used when
488 // doing stack walks) since the normal test is to check the return
489 // pc against the address saved below. so we may need to allow for
490 // this extra instruction in the check.
491
492 // save current address for use by exception handling code
493
494 return_address = __ pc();
495 entries.append(return_address);
496
497 // store result depending on type (everything that is not
498 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
499 // n.b. this assumes Java returns an integral result in r0
500 // and a floating result in j_farg0
501 __ ldr(j_rarg2, result);
502 Label is_long, is_float, is_double, exit;
503 __ ldr(j_rarg1, result_type);
504 __ cmp(j_rarg1, (u1)T_OBJECT);
505 __ br(Assembler::EQ, is_long);
506 __ cmp(j_rarg1, (u1)T_LONG);
507 __ br(Assembler::EQ, is_long);
508 __ cmp(j_rarg1, (u1)T_FLOAT);
509 __ br(Assembler::EQ, is_float);
510 __ cmp(j_rarg1, (u1)T_DOUBLE);
511 __ br(Assembler::EQ, is_double);
512
513 // handle T_INT case
514 __ strw(r0, Address(j_rarg2));
515
516 __ BIND(exit);
517
518 // pop parameters
519 __ sub(esp, rfp, -sp_after_call_off * wordSize);
520
521 #ifdef ASSERT
522 // verify that threads correspond
523 {
524 Label L, S;
525 __ ldr(rscratch1, thread);
526 __ cmp(rthread, rscratch1);
527 __ br(Assembler::NE, S);
528 __ get_thread(rscratch1);
529 __ cmp(rthread, rscratch1);
530 __ br(Assembler::EQ, L);
531 __ BIND(S);
532 __ stop("StubRoutines::call_stub: threads must correspond");
533 __ BIND(L);
534 }
546 __ ldp(r26, r25, r26_save);
547 __ ldp(r24, r23, r24_save);
548 __ ldp(r22, r21, r22_save);
549 __ ldp(r20, r19, r20_save);
550
551 // restore fpcr
552 __ ldr(rscratch1, fpcr_save);
553 __ set_fpcr(rscratch1);
554
555 __ ldp(c_rarg0, c_rarg1, call_wrapper);
556 __ ldrw(c_rarg2, result_type);
557 __ ldr(c_rarg3, method);
558 __ ldp(c_rarg4, c_rarg5, entry_point);
559 __ ldp(c_rarg6, c_rarg7, parameter_size);
560
561 // leave frame and return to caller
562 __ leave();
563 __ ret(lr);
564
565 // handle return types different from T_INT
566
567 __ BIND(is_long);
568 __ str(r0, Address(j_rarg2, 0));
569 __ br(Assembler::AL, exit);
570
571 __ BIND(is_float);
572 __ strs(j_farg0, Address(j_rarg2, 0));
573 __ br(Assembler::AL, exit);
574
575 __ BIND(is_double);
576 __ strd(j_farg0, Address(j_rarg2, 0));
577 __ br(Assembler::AL, exit);
578
579 // record the stub entry and end plus the auxiliary entry
580 store_archive_data(stub_id, start, __ pc(), &entries);
581
582 return start;
583 }
584
585 // Return point for a Java call if there's an exception thrown in
586 // Java code. The exception is caught and transformed into a
587 // pending exception stored in JavaThread that can be tested from
588 // within the VM.
589 //
590 // Note: Usually the parameters are removed by the callee. In case
591 // of an exception crossing an activation frame boundary, that is
592 // not the case if the callee is compiled code => need to setup the
593 // rsp.
594 //
595 // r0: exception oop
596
2583 // |array_tag| | header_size | element_type | |log2_element_size|
2584 // 32 30 24 16 8 2 0
2585 //
2586 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2587 //
2588
2589 const int lh_offset = in_bytes(Klass::layout_helper_offset());
2590
2591 // Handle objArrays completely differently...
2592 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2593 __ ldrw(lh, Address(scratch_src_klass, lh_offset));
2594 __ movw(rscratch1, objArray_lh);
2595 __ eorw(rscratch2, lh, rscratch1);
2596 __ cbzw(rscratch2, L_objArray);
2597
2598 // if (src->klass() != dst->klass()) return -1;
2599 __ load_klass(rscratch2, dst);
2600 __ eor(rscratch2, rscratch2, scratch_src_klass);
2601 __ cbnz(rscratch2, L_failed);
2602
2603 // if (!src->is_Array()) return -1;
2604 __ tbz(lh, 31, L_failed); // i.e. (lh >= 0)
2605
2606 // At this point, it is known to be a typeArray (array_tag 0x3).
2607 #ifdef ASSERT
2608 {
2609 BLOCK_COMMENT("assert primitive array {");
2610 Label L;
2611 __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2612 __ cmpw(lh, rscratch2);
2613 __ br(Assembler::GE, L);
2614 __ stop("must be a primitive array");
2615 __ bind(L);
2616 BLOCK_COMMENT("} assert primitive array done");
2617 }
2618 #endif
2619
2620 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2621 rscratch2, L_failed);
2622
11260 entries.append((address)aarch64_atomic_xchg_4_impl);
11261 entries.append((address)aarch64_atomic_xchg_8_impl);
11262 entries.append((address)aarch64_atomic_cmpxchg_1_impl);
11263 entries.append((address)aarch64_atomic_cmpxchg_4_impl);
11264 entries.append((address)aarch64_atomic_cmpxchg_8_impl);
11265 entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl);
11266 entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl);
11267 entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl);
11268 entries.append((address)aarch64_atomic_cmpxchg_4_release_impl);
11269 entries.append((address)aarch64_atomic_cmpxchg_8_release_impl);
11270 entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl);
11271 entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl);
11272
11273 assert(entries.length() == entry_count - 1,
11274 "unexpected extra entry count %d", entries.length());
11275
11276 store_archive_data(stub_id, start, end, &entries);
11277 }
11278 #endif // LINUX
11279
11280 address generate_cont_thaw(Continuation::thaw_kind kind) {
11281 bool return_barrier = Continuation::is_thaw_return_barrier(kind);
11282 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
11283
11284 address start = __ pc();
11285
11286 if (return_barrier) {
11287 __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
11288 __ mov(sp, rscratch1);
11289 }
11290 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
11291
11292 if (return_barrier) {
11293 // preserve possible return value from a method returning to the return barrier
11294 __ fmovd(rscratch1, v0);
11295 __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
11296 }
11297
11298 __ movw(c_rarg1, (return_barrier ? 1 : 0));
11299 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
11300 __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
11301
11302 if (return_barrier) {
11303 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
11304 __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
11305 __ fmovd(v0, rscratch1);
11306 }
11307 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
11308
11309
11310 Label thaw_success;
11311 // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
11312 __ cbnz(rscratch2, thaw_success);
11313 __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
11314 __ br(rscratch1);
11315 __ bind(thaw_success);
11316
11317 // make room for the thawed frames
11318 __ sub(rscratch1, sp, rscratch2);
11319 __ andr(rscratch1, rscratch1, -16); // align
11320 __ mov(sp, rscratch1);
11321
11322 if (return_barrier) {
11323 // save original return value -- again
11324 __ fmovd(rscratch1, v0);
11325 __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
11326 }
11327
11328 // If we want, we can templatize thaw by kind, and have three different entries
11329 __ movw(c_rarg1, (uint32_t)kind);
11330
11331 __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
11332 __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
11333
11334 if (return_barrier) {
11335 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
11336 __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
11337 __ fmovd(v0, rscratch1);
11338 } else {
11339 __ mov(r0, zr); // return 0 (success) from doYield
11340 }
11341
11342 // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
11343 __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
11344 __ mov(rfp, sp);
11345
11346 if (return_barrier_exception) {
11347 __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
11348 __ authenticate_return_address(c_rarg1);
11349 __ verify_oop(r0);
11350 // save return value containing the exception oop in callee-saved R19
11351 __ mov(r19, r0);
11352
11353 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
11354
11355 // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
11356 // __ reinitialize_ptrue();
11357
12522 // assert(Ra == Pa_base[j], "must be");
12523 // MACC(Ra, Ra, t0, t1, t2);
12524 // }
12525 // iters = (2*len-i)/2;
12526 // assert(iters == len-j, "must be");
12527 // for (; iters--; j++) {
12528 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
12529 // MACC(Rm, Rn, t0, t1, t2);
12530 // Rm = *++Pm;
12531 // Rn = *--Pn;
12532 // }
12533 // Pm_base[i-len] = t0;
12534 // t0 = t1; t1 = t2; t2 = 0;
12535 // }
12536
12537 // while (t0)
12538 // t0 = sub(Pm_base, Pn_base, t0, len);
12539 // }
12540 };
12541
12542 // Initialization
12543 void generate_preuniverse_stubs() {
12544 // preuniverse stubs are not needed for aarch64
12545 }
12546
12547 void generate_initial_stubs() {
12548 // Generate initial stubs and initializes the entry points
12549
12550 // entry points that exist in all platforms Note: This is code
12551 // that could be shared among different platforms - however the
12552 // benefit seems to be smaller than the disadvantage of having a
12553 // much more complicated generator structure. See also comment in
12554 // stubRoutines.hpp.
12555
12556 StubRoutines::_forward_exception_entry = generate_forward_exception();
12557
12558 StubRoutines::_call_stub_entry =
12559 generate_call_stub(StubRoutines::_call_stub_return_address);
12560
12561 // is referenced by megamorphic call
12570 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
12571 }
12572
12573 if (UseCRC32CIntrinsics) {
12574 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
12575 }
12576
12577 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
12578 StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
12579 }
12580
12581 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
12582 StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
12583 }
12584
12585 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
12586 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
12587 StubRoutines::_hf2f = generate_float16ToFloat();
12588 StubRoutines::_f2hf = generate_floatToFloat16();
12589 }
12590 }
12591
12592 void generate_continuation_stubs() {
12593 // Continuation stubs:
12594 StubRoutines::_cont_thaw = generate_cont_thaw();
12595 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
12596 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
12597 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
12598 }
12599
12600 void generate_final_stubs() {
12601 // support for verify_oop (must happen after universe_init)
12602 if (VerifyOops) {
12603 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
12604 }
12605
12606 // arraycopy stubs used by compilers
12607 generate_arraycopy_stubs();
12608
12609 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
|
481 __ blr(c_rarg4);
482
483 // we do this here because the notify will already have been done
484 // if we get to the next instruction via an exception
485 //
486 // n.b. adding this instruction here affects the calculation of
487 // whether or not a routine returns to the call stub (used when
488 // doing stack walks) since the normal test is to check the return
489 // pc against the address saved below. so we may need to allow for
490 // this extra instruction in the check.
491
492 // save current address for use by exception handling code
493
494 return_address = __ pc();
495 entries.append(return_address);
496
497 // store result depending on type (everything that is not
498 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
499 // n.b. this assumes Java returns an integral result in r0
500 // and a floating result in j_farg0
501 // All of j_rargN may be used to return inline type fields so be careful
502 // not to clobber those.
503 // SharedRuntime::generate_buffered_inline_type_adapter() knows the register
504 // assignment of Rresult below.
505 Register Rresult = r14, Rresult_type = r15;
506 __ ldr(Rresult, result);
507 Label is_long, is_float, is_double, check_prim, exit;
508 __ ldr(Rresult_type, result_type);
509 __ cmp(Rresult_type, (u1)T_OBJECT);
510 __ br(Assembler::EQ, check_prim);
511 __ cmp(Rresult_type, (u1)T_LONG);
512 __ br(Assembler::EQ, is_long);
513 __ cmp(Rresult_type, (u1)T_FLOAT);
514 __ br(Assembler::EQ, is_float);
515 __ cmp(Rresult_type, (u1)T_DOUBLE);
516 __ br(Assembler::EQ, is_double);
517
518 // handle T_INT case
519 __ strw(r0, Address(Rresult));
520
521 __ BIND(exit);
522
523 // pop parameters
524 __ sub(esp, rfp, -sp_after_call_off * wordSize);
525
526 #ifdef ASSERT
527 // verify that threads correspond
528 {
529 Label L, S;
530 __ ldr(rscratch1, thread);
531 __ cmp(rthread, rscratch1);
532 __ br(Assembler::NE, S);
533 __ get_thread(rscratch1);
534 __ cmp(rthread, rscratch1);
535 __ br(Assembler::EQ, L);
536 __ BIND(S);
537 __ stop("StubRoutines::call_stub: threads must correspond");
538 __ BIND(L);
539 }
551 __ ldp(r26, r25, r26_save);
552 __ ldp(r24, r23, r24_save);
553 __ ldp(r22, r21, r22_save);
554 __ ldp(r20, r19, r20_save);
555
556 // restore fpcr
557 __ ldr(rscratch1, fpcr_save);
558 __ set_fpcr(rscratch1);
559
560 __ ldp(c_rarg0, c_rarg1, call_wrapper);
561 __ ldrw(c_rarg2, result_type);
562 __ ldr(c_rarg3, method);
563 __ ldp(c_rarg4, c_rarg5, entry_point);
564 __ ldp(c_rarg6, c_rarg7, parameter_size);
565
566 // leave frame and return to caller
567 __ leave();
568 __ ret(lr);
569
570 // handle return types different from T_INT
571 __ BIND(check_prim);
572 if (InlineTypeReturnedAsFields) {
573 // Check for scalarized return value
574 __ tbz(r0, 0, is_long);
575 // Load pack handler address
576 __ andr(rscratch1, r0, -2);
577 __ ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
578 __ ldr(rscratch1, Address(rscratch1, InlineKlass::pack_handler_jobject_offset()));
579 __ blr(rscratch1);
580 __ b(exit);
581 }
582
583 __ BIND(is_long);
584 __ str(r0, Address(Rresult, 0));
585 __ br(Assembler::AL, exit);
586
587 __ BIND(is_float);
588 __ strs(j_farg0, Address(Rresult, 0));
589 __ br(Assembler::AL, exit);
590
591 __ BIND(is_double);
592 __ strd(j_farg0, Address(Rresult, 0));
593 __ br(Assembler::AL, exit);
594
595 // record the stub entry and end plus the auxiliary entry
596 store_archive_data(stub_id, start, __ pc(), &entries);
597
598 return start;
599 }
600
601 // Return point for a Java call if there's an exception thrown in
602 // Java code. The exception is caught and transformed into a
603 // pending exception stored in JavaThread that can be tested from
604 // within the VM.
605 //
606 // Note: Usually the parameters are removed by the callee. In case
607 // of an exception crossing an activation frame boundary, that is
608 // not the case if the callee is compiled code => need to setup the
609 // rsp.
610 //
611 // r0: exception oop
612
2599 // |array_tag| | header_size | element_type | |log2_element_size|
2600 // 32 30 24 16 8 2 0
2601 //
2602 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2603 //
2604
2605 const int lh_offset = in_bytes(Klass::layout_helper_offset());
2606
2607 // Handle objArrays completely differently...
2608 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2609 __ ldrw(lh, Address(scratch_src_klass, lh_offset));
2610 __ movw(rscratch1, objArray_lh);
2611 __ eorw(rscratch2, lh, rscratch1);
2612 __ cbzw(rscratch2, L_objArray);
2613
2614 // if (src->klass() != dst->klass()) return -1;
2615 __ load_klass(rscratch2, dst);
2616 __ eor(rscratch2, rscratch2, scratch_src_klass);
2617 __ cbnz(rscratch2, L_failed);
2618
2619 // Check for flat inline type array -> return -1
2620 __ test_flat_array_oop(src, rscratch2, L_failed);
2621
2622 // Check for null-free (non-flat) inline type array -> handle as object array
2623 __ test_null_free_array_oop(src, rscratch2, L_objArray);
2624
2625 // if (!src->is_Array()) return -1;
2626 __ tbz(lh, 31, L_failed); // i.e. (lh >= 0)
2627
2628 // At this point, it is known to be a typeArray (array_tag 0x3).
2629 #ifdef ASSERT
2630 {
2631 BLOCK_COMMENT("assert primitive array {");
2632 Label L;
2633 __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2634 __ cmpw(lh, rscratch2);
2635 __ br(Assembler::GE, L);
2636 __ stop("must be a primitive array");
2637 __ bind(L);
2638 BLOCK_COMMENT("} assert primitive array done");
2639 }
2640 #endif
2641
2642 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2643 rscratch2, L_failed);
2644
11282 entries.append((address)aarch64_atomic_xchg_4_impl);
11283 entries.append((address)aarch64_atomic_xchg_8_impl);
11284 entries.append((address)aarch64_atomic_cmpxchg_1_impl);
11285 entries.append((address)aarch64_atomic_cmpxchg_4_impl);
11286 entries.append((address)aarch64_atomic_cmpxchg_8_impl);
11287 entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl);
11288 entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl);
11289 entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl);
11290 entries.append((address)aarch64_atomic_cmpxchg_4_release_impl);
11291 entries.append((address)aarch64_atomic_cmpxchg_8_release_impl);
11292 entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl);
11293 entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl);
11294
11295 assert(entries.length() == entry_count - 1,
11296 "unexpected extra entry count %d", entries.length());
11297
11298 store_archive_data(stub_id, start, end, &entries);
11299 }
11300 #endif // LINUX
11301
11302 static void save_return_registers(MacroAssembler* masm) {
11303 if (InlineTypeReturnedAsFields) {
11304 masm->push(RegSet::range(r0, r7), sp);
11305 masm->sub(sp, sp, 4 * wordSize);
11306 masm->st1(v0, v1, v2, v3, masm->T1D, Address(sp));
11307 masm->sub(sp, sp, 4 * wordSize);
11308 masm->st1(v4, v5, v6, v7, masm->T1D, Address(sp));
11309 } else {
11310 masm->fmovd(rscratch1, v0);
11311 masm->stp(rscratch1, r0, Address(masm->pre(sp, -2 * wordSize)));
11312 }
11313 }
11314
11315 static void restore_return_registers(MacroAssembler* masm) {
11316 if (InlineTypeReturnedAsFields) {
11317 masm->ld1(v4, v5, v6, v7, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
11318 masm->ld1(v0, v1, v2, v3, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
11319 masm->pop(RegSet::range(r0, r7), sp);
11320 } else {
11321 masm->ldp(rscratch1, r0, Address(masm->post(sp, 2 * wordSize)));
11322 masm->fmovd(v0, rscratch1);
11323 }
11324 }
11325
11326 address generate_cont_thaw(Continuation::thaw_kind kind) {
11327 bool return_barrier = Continuation::is_thaw_return_barrier(kind);
11328 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
11329
11330 address start = __ pc();
11331
11332 if (return_barrier) {
11333 __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
11334 __ mov(sp, rscratch1);
11335 }
11336 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
11337
11338 if (return_barrier) {
11339 // preserve possible return value from a method returning to the return barrier
11340 save_return_registers(_masm);
11341 }
11342
11343 __ movw(c_rarg1, (return_barrier ? 1 : 0));
11344 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
11345 __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
11346
11347 if (return_barrier) {
11348 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
11349 restore_return_registers(_masm);
11350 }
11351 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
11352
11353
11354 Label thaw_success;
11355 // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
11356 __ cbnz(rscratch2, thaw_success);
11357 __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
11358 __ br(rscratch1);
11359 __ bind(thaw_success);
11360
11361 // make room for the thawed frames
11362 __ sub(rscratch1, sp, rscratch2);
11363 __ andr(rscratch1, rscratch1, -16); // align
11364 __ mov(sp, rscratch1);
11365
11366 if (return_barrier) {
11367 // save original return value -- again
11368 save_return_registers(_masm);
11369 }
11370
11371 // If we want, we can templatize thaw by kind, and have three different entries
11372 __ movw(c_rarg1, (uint32_t)kind);
11373
11374 __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
11375 __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
11376
11377 if (return_barrier) {
11378 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
11379 restore_return_registers(_masm);
11380 } else {
11381 __ mov(r0, zr); // return 0 (success) from doYield
11382 }
11383
11384 // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
11385 __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
11386 __ mov(rfp, sp);
11387
11388 if (return_barrier_exception) {
11389 __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
11390 __ authenticate_return_address(c_rarg1);
11391 __ verify_oop(r0);
11392 // save return value containing the exception oop in callee-saved R19
11393 __ mov(r19, r0);
11394
11395 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
11396
11397 // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
11398 // __ reinitialize_ptrue();
11399
12564 // assert(Ra == Pa_base[j], "must be");
12565 // MACC(Ra, Ra, t0, t1, t2);
12566 // }
12567 // iters = (2*len-i)/2;
12568 // assert(iters == len-j, "must be");
12569 // for (; iters--; j++) {
12570 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
12571 // MACC(Rm, Rn, t0, t1, t2);
12572 // Rm = *++Pm;
12573 // Rn = *--Pn;
12574 // }
12575 // Pm_base[i-len] = t0;
12576 // t0 = t1; t1 = t2; t2 = 0;
12577 // }
12578
12579 // while (t0)
12580 // t0 = sub(Pm_base, Pn_base, t0, len);
12581 // }
12582 };
12583
12584 // Call here from the interpreter or compiled code to either load
12585 // multiple returned values from the inline type instance being
12586 // returned to registers or to store returned values to a newly
12587 // allocated inline type instance.
12588 address generate_return_value_stub(address destination, const char* name, bool has_res) {
12589 // We need to save all registers the calling convention may use so
12590 // the runtime calls read or update those registers. This needs to
12591 // be in sync with SharedRuntime::java_return_convention().
12592 // n.b. aarch64 asserts that frame::arg_reg_save_area_bytes == 0
12593 enum layout {
12594 j_rarg7_off = 0, j_rarg7_2, // j_rarg7 is r0
12595 j_rarg6_off, j_rarg6_2,
12596 j_rarg5_off, j_rarg5_2,
12597 j_rarg4_off, j_rarg4_2,
12598 j_rarg3_off, j_rarg3_2,
12599 j_rarg2_off, j_rarg2_2,
12600 j_rarg1_off, j_rarg1_2,
12601 j_rarg0_off, j_rarg0_2,
12602
12603 j_farg7_off, j_farg7_2,
12604 j_farg6_off, j_farg6_2,
12605 j_farg5_off, j_farg5_2,
12606 j_farg4_off, j_farg4_2,
12607 j_farg3_off, j_farg3_2,
12608 j_farg2_off, j_farg2_2,
12609 j_farg1_off, j_farg1_2,
12610 j_farg0_off, j_farg0_2,
12611
12612 rfp_off, rfp_off2,
12613 return_off, return_off2,
12614
12615 framesize // inclusive of return address
12616 };
12617
12618 CodeBuffer code(name, 512, 64);
12619 MacroAssembler* masm = new MacroAssembler(&code);
12620
12621 int frame_size_in_bytes = align_up(framesize*BytesPerInt, 16);
12622 assert(frame_size_in_bytes == framesize*BytesPerInt, "misaligned");
12623 int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
12624 int frame_size_in_words = frame_size_in_bytes / wordSize;
12625
12626 OopMapSet* oop_maps = new OopMapSet();
12627 OopMap* map = new OopMap(frame_size_in_slots, 0);
12628
12629 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg7_off), j_rarg7->as_VMReg());
12630 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg6_off), j_rarg6->as_VMReg());
12631 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg5_off), j_rarg5->as_VMReg());
12632 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg4_off), j_rarg4->as_VMReg());
12633 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg3_off), j_rarg3->as_VMReg());
12634 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg2_off), j_rarg2->as_VMReg());
12635 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg1_off), j_rarg1->as_VMReg());
12636 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg0_off), j_rarg0->as_VMReg());
12637
12638 map->set_callee_saved(VMRegImpl::stack2reg(j_farg0_off), j_farg0->as_VMReg());
12639 map->set_callee_saved(VMRegImpl::stack2reg(j_farg1_off), j_farg1->as_VMReg());
12640 map->set_callee_saved(VMRegImpl::stack2reg(j_farg2_off), j_farg2->as_VMReg());
12641 map->set_callee_saved(VMRegImpl::stack2reg(j_farg3_off), j_farg3->as_VMReg());
12642 map->set_callee_saved(VMRegImpl::stack2reg(j_farg4_off), j_farg4->as_VMReg());
12643 map->set_callee_saved(VMRegImpl::stack2reg(j_farg5_off), j_farg5->as_VMReg());
12644 map->set_callee_saved(VMRegImpl::stack2reg(j_farg6_off), j_farg6->as_VMReg());
12645 map->set_callee_saved(VMRegImpl::stack2reg(j_farg7_off), j_farg7->as_VMReg());
12646
12647 address start = __ pc();
12648
12649 __ enter(); // Save FP and LR before call
12650
12651 __ stpd(j_farg1, j_farg0, Address(__ pre(sp, -2 * wordSize)));
12652 __ stpd(j_farg3, j_farg2, Address(__ pre(sp, -2 * wordSize)));
12653 __ stpd(j_farg5, j_farg4, Address(__ pre(sp, -2 * wordSize)));
12654 __ stpd(j_farg7, j_farg6, Address(__ pre(sp, -2 * wordSize)));
12655
12656 __ stp(j_rarg1, j_rarg0, Address(__ pre(sp, -2 * wordSize)));
12657 __ stp(j_rarg3, j_rarg2, Address(__ pre(sp, -2 * wordSize)));
12658 __ stp(j_rarg5, j_rarg4, Address(__ pre(sp, -2 * wordSize)));
12659 __ stp(j_rarg7, j_rarg6, Address(__ pre(sp, -2 * wordSize)));
12660
12661 int frame_complete = __ offset();
12662
12663 // Set up last_Java_sp and last_Java_fp
12664 address the_pc = __ pc();
12665 __ set_last_Java_frame(sp, noreg, the_pc, rscratch1);
12666
12667 // Call runtime
12668 __ mov(c_rarg1, r0);
12669 __ mov(c_rarg0, rthread);
12670
12671 __ mov(rscratch1, destination);
12672 __ blr(rscratch1);
12673
12674 oop_maps->add_gc_map(the_pc - start, map);
12675
12676 __ reset_last_Java_frame(false);
12677
12678 __ ldp(j_rarg7, j_rarg6, Address(__ post(sp, 2 * wordSize)));
12679 __ ldp(j_rarg5, j_rarg4, Address(__ post(sp, 2 * wordSize)));
12680 __ ldp(j_rarg3, j_rarg2, Address(__ post(sp, 2 * wordSize)));
12681 __ ldp(j_rarg1, j_rarg0, Address(__ post(sp, 2 * wordSize)));
12682
12683 __ ldpd(j_farg7, j_farg6, Address(__ post(sp, 2 * wordSize)));
12684 __ ldpd(j_farg5, j_farg4, Address(__ post(sp, 2 * wordSize)));
12685 __ ldpd(j_farg3, j_farg2, Address(__ post(sp, 2 * wordSize)));
12686 __ ldpd(j_farg1, j_farg0, Address(__ post(sp, 2 * wordSize)));
12687
12688 __ leave();
12689
12690 // check for pending exceptions
12691 Label pending;
12692 __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
12693 __ cbnz(rscratch1, pending);
12694
12695 if (has_res) {
12696 __ get_vm_result_oop(r0, rthread);
12697 }
12698
12699 __ ret(lr);
12700
12701 __ bind(pending);
12702 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
12703
12704 // -------------
12705 // make sure all code is generated
12706 masm->flush();
12707
12708 RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, frame_size_in_words, oop_maps, false);
12709 return stub->entry_point();
12710 }
12711
12712 // Initialization
12713 void generate_preuniverse_stubs() {
12714 // preuniverse stubs are not needed for aarch64
12715 }
12716
12717 void generate_initial_stubs() {
12718 // Generate initial stubs and initializes the entry points
12719
12720 // entry points that exist in all platforms Note: This is code
12721 // that could be shared among different platforms - however the
12722 // benefit seems to be smaller than the disadvantage of having a
12723 // much more complicated generator structure. See also comment in
12724 // stubRoutines.hpp.
12725
12726 StubRoutines::_forward_exception_entry = generate_forward_exception();
12727
12728 StubRoutines::_call_stub_entry =
12729 generate_call_stub(StubRoutines::_call_stub_return_address);
12730
12731 // is referenced by megamorphic call
12740 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
12741 }
12742
12743 if (UseCRC32CIntrinsics) {
12744 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
12745 }
12746
12747 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
12748 StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
12749 }
12750
12751 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
12752 StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
12753 }
12754
12755 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
12756 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
12757 StubRoutines::_hf2f = generate_float16ToFloat();
12758 StubRoutines::_f2hf = generate_floatToFloat16();
12759 }
12760
12761 if (InlineTypeReturnedAsFields) {
12762 StubRoutines::_load_inline_type_fields_in_regs =
12763 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::load_inline_type_fields_in_regs), "load_inline_type_fields_in_regs", false);
12764 StubRoutines::_store_inline_type_fields_to_buf =
12765 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::store_inline_type_fields_to_buf), "store_inline_type_fields_to_buf", true);
12766 }
12767
12768 }
12769
12770 void generate_continuation_stubs() {
12771 // Continuation stubs:
12772 StubRoutines::_cont_thaw = generate_cont_thaw();
12773 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
12774 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
12775 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
12776 }
12777
12778 void generate_final_stubs() {
12779 // support for verify_oop (must happen after universe_init)
12780 if (VerifyOops) {
12781 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
12782 }
12783
12784 // arraycopy stubs used by compilers
12785 generate_arraycopy_stubs();
12786
12787 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
|