481 __ blr(c_rarg4);
482
483 // we do this here because the notify will already have been done
484 // if we get to the next instruction via an exception
485 //
486 // n.b. adding this instruction here affects the calculation of
487 // whether or not a routine returns to the call stub (used when
488 // doing stack walks) since the normal test is to check the return
489 // pc against the address saved below. so we may need to allow for
490 // this extra instruction in the check.
491
492 // save current address for use by exception handling code
493
494 return_address = __ pc();
495 entries.append(return_address);
496
497 // store result depending on type (everything that is not
498 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
499 // n.b. this assumes Java returns an integral result in r0
500 // and a floating result in j_farg0
501 __ ldr(j_rarg2, result);
502 Label is_long, is_float, is_double, exit;
503 __ ldr(j_rarg1, result_type);
504 __ cmp(j_rarg1, (u1)T_OBJECT);
505 __ br(Assembler::EQ, is_long);
506 __ cmp(j_rarg1, (u1)T_LONG);
507 __ br(Assembler::EQ, is_long);
508 __ cmp(j_rarg1, (u1)T_FLOAT);
509 __ br(Assembler::EQ, is_float);
510 __ cmp(j_rarg1, (u1)T_DOUBLE);
511 __ br(Assembler::EQ, is_double);
512
513 // handle T_INT case
514 __ strw(r0, Address(j_rarg2));
515
516 __ BIND(exit);
517
518 // pop parameters
519 __ sub(esp, rfp, -sp_after_call_off * wordSize);
520
521 #ifdef ASSERT
522 // verify that threads correspond
523 {
524 Label L, S;
525 __ ldr(rscratch1, thread);
526 __ cmp(rthread, rscratch1);
527 __ br(Assembler::NE, S);
528 __ get_thread(rscratch1);
529 __ cmp(rthread, rscratch1);
530 __ br(Assembler::EQ, L);
531 __ BIND(S);
532 __ stop("StubRoutines::call_stub: threads must correspond");
533 __ BIND(L);
534 }
546 __ ldp(r26, r25, r26_save);
547 __ ldp(r24, r23, r24_save);
548 __ ldp(r22, r21, r22_save);
549 __ ldp(r20, r19, r20_save);
550
551 // restore fpcr
552 __ ldr(rscratch1, fpcr_save);
553 __ set_fpcr(rscratch1);
554
555 __ ldp(c_rarg0, c_rarg1, call_wrapper);
556 __ ldrw(c_rarg2, result_type);
557 __ ldr(c_rarg3, method);
558 __ ldp(c_rarg4, c_rarg5, entry_point);
559 __ ldp(c_rarg6, c_rarg7, parameter_size);
560
561 // leave frame and return to caller
562 __ leave();
563 __ ret(lr);
564
565 // handle return types different from T_INT
566
567 __ BIND(is_long);
568 __ str(r0, Address(j_rarg2, 0));
569 __ br(Assembler::AL, exit);
570
571 __ BIND(is_float);
572 __ strs(j_farg0, Address(j_rarg2, 0));
573 __ br(Assembler::AL, exit);
574
575 __ BIND(is_double);
576 __ strd(j_farg0, Address(j_rarg2, 0));
577 __ br(Assembler::AL, exit);
578
579 // record the stub entry and end plus the auxiliary entry
580 store_archive_data(stub_id, start, __ pc(), &entries);
581
582 return start;
583 }
584
585 // Return point for a Java call if there's an exception thrown in
586 // Java code. The exception is caught and transformed into a
587 // pending exception stored in JavaThread that can be tested from
588 // within the VM.
589 //
590 // Note: Usually the parameters are removed by the callee. In case
591 // of an exception crossing an activation frame boundary, that is
592 // not the case if the callee is compiled code => need to setup the
593 // rsp.
594 //
595 // r0: exception oop
596
2583 // |array_tag| | header_size | element_type | |log2_element_size|
2584 // 32 30 24 16 8 2 0
2585 //
2586 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2587 //
2588
2589 const int lh_offset = in_bytes(Klass::layout_helper_offset());
2590
2591 // Handle objArrays completely differently...
2592 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2593 __ ldrw(lh, Address(scratch_src_klass, lh_offset));
2594 __ movw(rscratch1, objArray_lh);
2595 __ eorw(rscratch2, lh, rscratch1);
2596 __ cbzw(rscratch2, L_objArray);
2597
2598 // if (src->klass() != dst->klass()) return -1;
2599 __ load_klass(rscratch2, dst);
2600 __ eor(rscratch2, rscratch2, scratch_src_klass);
2601 __ cbnz(rscratch2, L_failed);
2602
2603 // if (!src->is_Array()) return -1;
2604 __ tbz(lh, 31, L_failed); // i.e. (lh >= 0)
2605
2606 // At this point, it is known to be a typeArray (array_tag 0x3).
2607 #ifdef ASSERT
2608 {
2609 BLOCK_COMMENT("assert primitive array {");
2610 Label L;
2611 __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2612 __ cmpw(lh, rscratch2);
2613 __ br(Assembler::GE, L);
2614 __ stop("must be a primitive array");
2615 __ bind(L);
2616 BLOCK_COMMENT("} assert primitive array done");
2617 }
2618 #endif
2619
2620 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2621 rscratch2, L_failed);
2622
11278 entries.append((address)aarch64_atomic_xchg_4_impl);
11279 entries.append((address)aarch64_atomic_xchg_8_impl);
11280 entries.append((address)aarch64_atomic_cmpxchg_1_impl);
11281 entries.append((address)aarch64_atomic_cmpxchg_4_impl);
11282 entries.append((address)aarch64_atomic_cmpxchg_8_impl);
11283 entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl);
11284 entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl);
11285 entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl);
11286 entries.append((address)aarch64_atomic_cmpxchg_4_release_impl);
11287 entries.append((address)aarch64_atomic_cmpxchg_8_release_impl);
11288 entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl);
11289 entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl);
11290
11291 assert(entries.length() == entry_count - 1,
11292 "unexpected extra entry count %d", entries.length());
11293
11294 store_archive_data(stub_id, start, end, &entries);
11295 }
11296 #endif // LINUX
11297
11298 address generate_cont_thaw(Continuation::thaw_kind kind) {
11299 bool return_barrier = Continuation::is_thaw_return_barrier(kind);
11300 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
11301
11302 address start = __ pc();
11303
11304 if (return_barrier) {
11305 __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
11306 __ mov(sp, rscratch1);
11307 }
11308 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
11309
11310 if (return_barrier) {
11311 // preserve possible return value from a method returning to the return barrier
11312 __ fmovd(rscratch1, v0);
11313 __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
11314 }
11315
11316 __ movw(c_rarg1, (return_barrier ? 1 : 0));
11317 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
11318 __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
11319
11320 if (return_barrier) {
11321 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
11322 __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
11323 __ fmovd(v0, rscratch1);
11324 }
11325 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
11326
11327
11328 Label thaw_success;
11329 // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
11330 __ cbnz(rscratch2, thaw_success);
11331 __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
11332 __ br(rscratch1);
11333 __ bind(thaw_success);
11334
11335 // make room for the thawed frames
11336 __ sub(rscratch1, sp, rscratch2);
11337 __ andr(rscratch1, rscratch1, -16); // align
11338 __ mov(sp, rscratch1);
11339
11340 if (return_barrier) {
11341 // save original return value -- again
11342 __ fmovd(rscratch1, v0);
11343 __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
11344 }
11345
11346 // If we want, we can templatize thaw by kind, and have three different entries
11347 __ movw(c_rarg1, (uint32_t)kind);
11348
11349 __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
11350 __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
11351
11352 if (return_barrier) {
11353 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
11354 __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
11355 __ fmovd(v0, rscratch1);
11356 } else {
11357 __ mov(r0, zr); // return 0 (success) from doYield
11358 }
11359
11360 // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
11361 __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
11362 __ mov(rfp, sp);
11363
11364 if (return_barrier_exception) {
11365 __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
11366 __ authenticate_return_address(c_rarg1);
11367 __ verify_oop(r0);
11368 // save return value containing the exception oop in callee-saved R19
11369 __ mov(r19, r0);
11370
11371 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
11372
11373 // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
11374 // __ reinitialize_ptrue();
11375
12540 // assert(Ra == Pa_base[j], "must be");
12541 // MACC(Ra, Ra, t0, t1, t2);
12542 // }
12543 // iters = (2*len-i)/2;
12544 // assert(iters == len-j, "must be");
12545 // for (; iters--; j++) {
12546 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
12547 // MACC(Rm, Rn, t0, t1, t2);
12548 // Rm = *++Pm;
12549 // Rn = *--Pn;
12550 // }
12551 // Pm_base[i-len] = t0;
12552 // t0 = t1; t1 = t2; t2 = 0;
12553 // }
12554
12555 // while (t0)
12556 // t0 = sub(Pm_base, Pn_base, t0, len);
12557 // }
12558 };
12559
12560 // Initialization
12561 void generate_preuniverse_stubs() {
12562 // preuniverse stubs are not needed for aarch64
12563 }
12564
12565 void generate_initial_stubs() {
12566 // Generate initial stubs and initializes the entry points
12567
12568 // entry points that exist in all platforms Note: This is code
12569 // that could be shared among different platforms - however the
12570 // benefit seems to be smaller than the disadvantage of having a
12571 // much more complicated generator structure. See also comment in
12572 // stubRoutines.hpp.
12573
12574 StubRoutines::_forward_exception_entry = generate_forward_exception();
12575
12576 StubRoutines::_call_stub_entry =
12577 generate_call_stub(StubRoutines::_call_stub_return_address);
12578
12579 // is referenced by megamorphic call
12588 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
12589 }
12590
12591 if (UseCRC32CIntrinsics) {
12592 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
12593 }
12594
12595 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
12596 StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
12597 }
12598
12599 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
12600 StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
12601 }
12602
12603 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
12604 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
12605 StubRoutines::_hf2f = generate_float16ToFloat();
12606 StubRoutines::_f2hf = generate_floatToFloat16();
12607 }
12608 }
12609
12610 void generate_continuation_stubs() {
12611 // Continuation stubs:
12612 StubRoutines::_cont_thaw = generate_cont_thaw();
12613 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
12614 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
12615 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
12616 }
12617
12618 void generate_final_stubs() {
12619 // support for verify_oop (must happen after universe_init)
12620 if (VerifyOops) {
12621 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
12622 }
12623
12624 // arraycopy stubs used by compilers
12625 generate_arraycopy_stubs();
12626
12627 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
|
481 __ blr(c_rarg4);
482
483 // we do this here because the notify will already have been done
484 // if we get to the next instruction via an exception
485 //
486 // n.b. adding this instruction here affects the calculation of
487 // whether or not a routine returns to the call stub (used when
488 // doing stack walks) since the normal test is to check the return
489 // pc against the address saved below. so we may need to allow for
490 // this extra instruction in the check.
491
492 // save current address for use by exception handling code
493
494 return_address = __ pc();
495 entries.append(return_address);
496
497 // store result depending on type (everything that is not
498 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
499 // n.b. this assumes Java returns an integral result in r0
500 // and a floating result in j_farg0
501 // All of j_rargN may be used to return inline type fields so be careful
502 // not to clobber those.
503 // SharedRuntime::generate_buffered_inline_type_adapter() knows the register
504 // assignment of Rresult below.
505 Register Rresult = r14, Rresult_type = r15;
506 __ ldr(Rresult, result);
507 Label is_long, is_float, is_double, check_prim, exit;
508 __ ldr(Rresult_type, result_type);
509 __ cmp(Rresult_type, (u1)T_OBJECT);
510 __ br(Assembler::EQ, check_prim);
511 __ cmp(Rresult_type, (u1)T_LONG);
512 __ br(Assembler::EQ, is_long);
513 __ cmp(Rresult_type, (u1)T_FLOAT);
514 __ br(Assembler::EQ, is_float);
515 __ cmp(Rresult_type, (u1)T_DOUBLE);
516 __ br(Assembler::EQ, is_double);
517
518 // handle T_INT case
519 __ strw(r0, Address(Rresult));
520
521 __ BIND(exit);
522
523 // pop parameters
524 __ sub(esp, rfp, -sp_after_call_off * wordSize);
525
526 #ifdef ASSERT
527 // verify that threads correspond
528 {
529 Label L, S;
530 __ ldr(rscratch1, thread);
531 __ cmp(rthread, rscratch1);
532 __ br(Assembler::NE, S);
533 __ get_thread(rscratch1);
534 __ cmp(rthread, rscratch1);
535 __ br(Assembler::EQ, L);
536 __ BIND(S);
537 __ stop("StubRoutines::call_stub: threads must correspond");
538 __ BIND(L);
539 }
551 __ ldp(r26, r25, r26_save);
552 __ ldp(r24, r23, r24_save);
553 __ ldp(r22, r21, r22_save);
554 __ ldp(r20, r19, r20_save);
555
556 // restore fpcr
557 __ ldr(rscratch1, fpcr_save);
558 __ set_fpcr(rscratch1);
559
560 __ ldp(c_rarg0, c_rarg1, call_wrapper);
561 __ ldrw(c_rarg2, result_type);
562 __ ldr(c_rarg3, method);
563 __ ldp(c_rarg4, c_rarg5, entry_point);
564 __ ldp(c_rarg6, c_rarg7, parameter_size);
565
566 // leave frame and return to caller
567 __ leave();
568 __ ret(lr);
569
570 // handle return types different from T_INT
571 __ BIND(check_prim);
572 if (InlineTypeReturnedAsFields) {
573 // Check for scalarized return value
574 __ tbz(r0, 0, is_long);
575 // Load pack handler address
576 __ andr(rscratch1, r0, -2);
577 __ ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
578 __ ldr(rscratch1, Address(rscratch1, InlineKlass::pack_handler_jobject_offset()));
579 __ blr(rscratch1);
580 __ b(exit);
581 }
582
583 __ BIND(is_long);
584 __ str(r0, Address(Rresult, 0));
585 __ br(Assembler::AL, exit);
586
587 __ BIND(is_float);
588 __ strs(j_farg0, Address(Rresult, 0));
589 __ br(Assembler::AL, exit);
590
591 __ BIND(is_double);
592 __ strd(j_farg0, Address(Rresult, 0));
593 __ br(Assembler::AL, exit);
594
595 // record the stub entry and end plus the auxiliary entry
596 store_archive_data(stub_id, start, __ pc(), &entries);
597
598 return start;
599 }
600
601 // Return point for a Java call if there's an exception thrown in
602 // Java code. The exception is caught and transformed into a
603 // pending exception stored in JavaThread that can be tested from
604 // within the VM.
605 //
606 // Note: Usually the parameters are removed by the callee. In case
607 // of an exception crossing an activation frame boundary, that is
608 // not the case if the callee is compiled code => need to setup the
609 // rsp.
610 //
611 // r0: exception oop
612
2599 // |array_tag| | header_size | element_type | |log2_element_size|
2600 // 32 30 24 16 8 2 0
2601 //
2602 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2603 //
2604
2605 const int lh_offset = in_bytes(Klass::layout_helper_offset());
2606
2607 // Handle objArrays completely differently...
2608 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2609 __ ldrw(lh, Address(scratch_src_klass, lh_offset));
2610 __ movw(rscratch1, objArray_lh);
2611 __ eorw(rscratch2, lh, rscratch1);
2612 __ cbzw(rscratch2, L_objArray);
2613
2614 // if (src->klass() != dst->klass()) return -1;
2615 __ load_klass(rscratch2, dst);
2616 __ eor(rscratch2, rscratch2, scratch_src_klass);
2617 __ cbnz(rscratch2, L_failed);
2618
2619 // Check for flat inline type array -> return -1
2620 __ test_flat_array_oop(src, rscratch2, L_failed);
2621
2622 // Check for null-free (non-flat) inline type array -> handle as object array
2623 __ test_null_free_array_oop(src, rscratch2, L_objArray);
2624
2625 // if (!src->is_Array()) return -1;
2626 __ tbz(lh, 31, L_failed); // i.e. (lh >= 0)
2627
2628 // At this point, it is known to be a typeArray (array_tag 0x3).
2629 #ifdef ASSERT
2630 {
2631 BLOCK_COMMENT("assert primitive array {");
2632 Label L;
2633 __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2634 __ cmpw(lh, rscratch2);
2635 __ br(Assembler::GE, L);
2636 __ stop("must be a primitive array");
2637 __ bind(L);
2638 BLOCK_COMMENT("} assert primitive array done");
2639 }
2640 #endif
2641
2642 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2643 rscratch2, L_failed);
2644
11300 entries.append((address)aarch64_atomic_xchg_4_impl);
11301 entries.append((address)aarch64_atomic_xchg_8_impl);
11302 entries.append((address)aarch64_atomic_cmpxchg_1_impl);
11303 entries.append((address)aarch64_atomic_cmpxchg_4_impl);
11304 entries.append((address)aarch64_atomic_cmpxchg_8_impl);
11305 entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl);
11306 entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl);
11307 entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl);
11308 entries.append((address)aarch64_atomic_cmpxchg_4_release_impl);
11309 entries.append((address)aarch64_atomic_cmpxchg_8_release_impl);
11310 entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl);
11311 entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl);
11312
11313 assert(entries.length() == entry_count - 1,
11314 "unexpected extra entry count %d", entries.length());
11315
11316 store_archive_data(stub_id, start, end, &entries);
11317 }
11318 #endif // LINUX
11319
11320 static void save_return_registers(MacroAssembler* masm) {
11321 if (InlineTypeReturnedAsFields) {
11322 masm->push(RegSet::range(r0, r7), sp);
11323 masm->sub(sp, sp, 4 * wordSize);
11324 masm->st1(v0, v1, v2, v3, masm->T1D, Address(sp));
11325 masm->sub(sp, sp, 4 * wordSize);
11326 masm->st1(v4, v5, v6, v7, masm->T1D, Address(sp));
11327 } else {
11328 masm->fmovd(rscratch1, v0);
11329 masm->stp(rscratch1, r0, Address(masm->pre(sp, -2 * wordSize)));
11330 }
11331 }
11332
11333 static void restore_return_registers(MacroAssembler* masm) {
11334 if (InlineTypeReturnedAsFields) {
11335 masm->ld1(v4, v5, v6, v7, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
11336 masm->ld1(v0, v1, v2, v3, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
11337 masm->pop(RegSet::range(r0, r7), sp);
11338 } else {
11339 masm->ldp(rscratch1, r0, Address(masm->post(sp, 2 * wordSize)));
11340 masm->fmovd(v0, rscratch1);
11341 }
11342 }
11343
11344 address generate_cont_thaw(Continuation::thaw_kind kind) {
11345 bool return_barrier = Continuation::is_thaw_return_barrier(kind);
11346 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
11347
11348 address start = __ pc();
11349
11350 if (return_barrier) {
11351 __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
11352 __ mov(sp, rscratch1);
11353 }
11354 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
11355
11356 if (return_barrier) {
11357 // preserve possible return value from a method returning to the return barrier
11358 save_return_registers(_masm);
11359 }
11360
11361 __ movw(c_rarg1, (return_barrier ? 1 : 0));
11362 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
11363 __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
11364
11365 if (return_barrier) {
11366 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
11367 restore_return_registers(_masm);
11368 }
11369 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
11370
11371
11372 Label thaw_success;
11373 // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
11374 __ cbnz(rscratch2, thaw_success);
11375 __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
11376 __ br(rscratch1);
11377 __ bind(thaw_success);
11378
11379 // make room for the thawed frames
11380 __ sub(rscratch1, sp, rscratch2);
11381 __ andr(rscratch1, rscratch1, -16); // align
11382 __ mov(sp, rscratch1);
11383
11384 if (return_barrier) {
11385 // save original return value -- again
11386 save_return_registers(_masm);
11387 }
11388
11389 // If we want, we can templatize thaw by kind, and have three different entries
11390 __ movw(c_rarg1, (uint32_t)kind);
11391
11392 __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
11393 __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
11394
11395 if (return_barrier) {
11396 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
11397 restore_return_registers(_masm);
11398 } else {
11399 __ mov(r0, zr); // return 0 (success) from doYield
11400 }
11401
11402 // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
11403 __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
11404 __ mov(rfp, sp);
11405
11406 if (return_barrier_exception) {
11407 __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
11408 __ authenticate_return_address(c_rarg1);
11409 __ verify_oop(r0);
11410 // save return value containing the exception oop in callee-saved R19
11411 __ mov(r19, r0);
11412
11413 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
11414
11415 // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
11416 // __ reinitialize_ptrue();
11417
12582 // assert(Ra == Pa_base[j], "must be");
12583 // MACC(Ra, Ra, t0, t1, t2);
12584 // }
12585 // iters = (2*len-i)/2;
12586 // assert(iters == len-j, "must be");
12587 // for (; iters--; j++) {
12588 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
12589 // MACC(Rm, Rn, t0, t1, t2);
12590 // Rm = *++Pm;
12591 // Rn = *--Pn;
12592 // }
12593 // Pm_base[i-len] = t0;
12594 // t0 = t1; t1 = t2; t2 = 0;
12595 // }
12596
12597 // while (t0)
12598 // t0 = sub(Pm_base, Pn_base, t0, len);
12599 // }
12600 };
12601
12602 // Call here from the interpreter or compiled code to either load
12603 // multiple returned values from the inline type instance being
12604 // returned to registers or to store returned values to a newly
12605 // allocated inline type instance.
12606 address generate_return_value_stub(address destination, const char* name, bool has_res) {
12607 // We need to save all registers the calling convention may use so
12608 // the runtime calls read or update those registers. This needs to
12609 // be in sync with SharedRuntime::java_return_convention().
12610 // n.b. aarch64 asserts that frame::arg_reg_save_area_bytes == 0
12611 enum layout {
12612 j_rarg7_off = 0, j_rarg7_2, // j_rarg7 is r0
12613 j_rarg6_off, j_rarg6_2,
12614 j_rarg5_off, j_rarg5_2,
12615 j_rarg4_off, j_rarg4_2,
12616 j_rarg3_off, j_rarg3_2,
12617 j_rarg2_off, j_rarg2_2,
12618 j_rarg1_off, j_rarg1_2,
12619 j_rarg0_off, j_rarg0_2,
12620
12621 j_farg7_off, j_farg7_2,
12622 j_farg6_off, j_farg6_2,
12623 j_farg5_off, j_farg5_2,
12624 j_farg4_off, j_farg4_2,
12625 j_farg3_off, j_farg3_2,
12626 j_farg2_off, j_farg2_2,
12627 j_farg1_off, j_farg1_2,
12628 j_farg0_off, j_farg0_2,
12629
12630 rfp_off, rfp_off2,
12631 return_off, return_off2,
12632
12633 framesize // inclusive of return address
12634 };
12635
12636 CodeBuffer code(name, 512, 64);
12637 MacroAssembler* masm = new MacroAssembler(&code);
12638
12639 int frame_size_in_bytes = align_up(framesize*BytesPerInt, 16);
12640 assert(frame_size_in_bytes == framesize*BytesPerInt, "misaligned");
12641 int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
12642 int frame_size_in_words = frame_size_in_bytes / wordSize;
12643
12644 OopMapSet* oop_maps = new OopMapSet();
12645 OopMap* map = new OopMap(frame_size_in_slots, 0);
12646
12647 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg7_off), j_rarg7->as_VMReg());
12648 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg6_off), j_rarg6->as_VMReg());
12649 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg5_off), j_rarg5->as_VMReg());
12650 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg4_off), j_rarg4->as_VMReg());
12651 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg3_off), j_rarg3->as_VMReg());
12652 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg2_off), j_rarg2->as_VMReg());
12653 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg1_off), j_rarg1->as_VMReg());
12654 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg0_off), j_rarg0->as_VMReg());
12655
12656 map->set_callee_saved(VMRegImpl::stack2reg(j_farg0_off), j_farg0->as_VMReg());
12657 map->set_callee_saved(VMRegImpl::stack2reg(j_farg1_off), j_farg1->as_VMReg());
12658 map->set_callee_saved(VMRegImpl::stack2reg(j_farg2_off), j_farg2->as_VMReg());
12659 map->set_callee_saved(VMRegImpl::stack2reg(j_farg3_off), j_farg3->as_VMReg());
12660 map->set_callee_saved(VMRegImpl::stack2reg(j_farg4_off), j_farg4->as_VMReg());
12661 map->set_callee_saved(VMRegImpl::stack2reg(j_farg5_off), j_farg5->as_VMReg());
12662 map->set_callee_saved(VMRegImpl::stack2reg(j_farg6_off), j_farg6->as_VMReg());
12663 map->set_callee_saved(VMRegImpl::stack2reg(j_farg7_off), j_farg7->as_VMReg());
12664
12665 address start = __ pc();
12666
12667 __ enter(); // Save FP and LR before call
12668
12669 __ stpd(j_farg1, j_farg0, Address(__ pre(sp, -2 * wordSize)));
12670 __ stpd(j_farg3, j_farg2, Address(__ pre(sp, -2 * wordSize)));
12671 __ stpd(j_farg5, j_farg4, Address(__ pre(sp, -2 * wordSize)));
12672 __ stpd(j_farg7, j_farg6, Address(__ pre(sp, -2 * wordSize)));
12673
12674 __ stp(j_rarg1, j_rarg0, Address(__ pre(sp, -2 * wordSize)));
12675 __ stp(j_rarg3, j_rarg2, Address(__ pre(sp, -2 * wordSize)));
12676 __ stp(j_rarg5, j_rarg4, Address(__ pre(sp, -2 * wordSize)));
12677 __ stp(j_rarg7, j_rarg6, Address(__ pre(sp, -2 * wordSize)));
12678
12679 int frame_complete = __ offset();
12680
12681 // Set up last_Java_sp and last_Java_fp
12682 address the_pc = __ pc();
12683 __ set_last_Java_frame(sp, noreg, the_pc, rscratch1);
12684
12685 // Call runtime
12686 __ mov(c_rarg1, r0);
12687 __ mov(c_rarg0, rthread);
12688
12689 __ mov(rscratch1, destination);
12690 __ blr(rscratch1);
12691
12692 oop_maps->add_gc_map(the_pc - start, map);
12693
12694 __ reset_last_Java_frame(false);
12695
12696 __ ldp(j_rarg7, j_rarg6, Address(__ post(sp, 2 * wordSize)));
12697 __ ldp(j_rarg5, j_rarg4, Address(__ post(sp, 2 * wordSize)));
12698 __ ldp(j_rarg3, j_rarg2, Address(__ post(sp, 2 * wordSize)));
12699 __ ldp(j_rarg1, j_rarg0, Address(__ post(sp, 2 * wordSize)));
12700
12701 __ ldpd(j_farg7, j_farg6, Address(__ post(sp, 2 * wordSize)));
12702 __ ldpd(j_farg5, j_farg4, Address(__ post(sp, 2 * wordSize)));
12703 __ ldpd(j_farg3, j_farg2, Address(__ post(sp, 2 * wordSize)));
12704 __ ldpd(j_farg1, j_farg0, Address(__ post(sp, 2 * wordSize)));
12705
12706 // check for pending exceptions
12707 Label pending;
12708 __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
12709 __ cbnz(rscratch1, pending);
12710
12711 if (has_res) {
12712 // We just called SharedRuntime::store_inline_type_fields_to_buf. Check if we still
12713 // need to initialize the buffer and if so, call the inline class specific pack handler.
12714 Label skip_pack;
12715 __ get_vm_result_oop(r0, rthread);
12716 __ get_vm_result_metadata(rscratch1, rthread);
12717 __ cbz(rscratch1, skip_pack);
12718 __ ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
12719 __ ldr(rscratch1, Address(rscratch1, InlineKlass::pack_handler_offset()));
12720 __ blr(rscratch1);
12721 __ membar(Assembler::StoreStore);
12722 __ bind(skip_pack);
12723 }
12724
12725 __ leave();
12726 __ ret(lr);
12727
12728 __ bind(pending);
12729 __ leave();
12730 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
12731
12732 // -------------
12733 // make sure all code is generated
12734 masm->flush();
12735
12736 RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, frame_size_in_words, oop_maps, false);
12737 return stub->entry_point();
12738 }
12739
12740 // Initialization
12741 void generate_preuniverse_stubs() {
12742 // preuniverse stubs are not needed for aarch64
12743 }
12744
12745 void generate_initial_stubs() {
12746 // Generate initial stubs and initializes the entry points
12747
12748 // entry points that exist in all platforms Note: This is code
12749 // that could be shared among different platforms - however the
12750 // benefit seems to be smaller than the disadvantage of having a
12751 // much more complicated generator structure. See also comment in
12752 // stubRoutines.hpp.
12753
12754 StubRoutines::_forward_exception_entry = generate_forward_exception();
12755
12756 StubRoutines::_call_stub_entry =
12757 generate_call_stub(StubRoutines::_call_stub_return_address);
12758
12759 // is referenced by megamorphic call
12768 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
12769 }
12770
12771 if (UseCRC32CIntrinsics) {
12772 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
12773 }
12774
12775 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
12776 StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
12777 }
12778
12779 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
12780 StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
12781 }
12782
12783 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
12784 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
12785 StubRoutines::_hf2f = generate_float16ToFloat();
12786 StubRoutines::_f2hf = generate_floatToFloat16();
12787 }
12788
12789 if (InlineTypeReturnedAsFields) {
12790 StubRoutines::_load_inline_type_fields_in_regs =
12791 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::load_inline_type_fields_in_regs), "load_inline_type_fields_in_regs", false);
12792 StubRoutines::_store_inline_type_fields_to_buf =
12793 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::store_inline_type_fields_to_buf), "store_inline_type_fields_to_buf", true);
12794 }
12795
12796 }
12797
12798 void generate_continuation_stubs() {
12799 // Continuation stubs:
12800 StubRoutines::_cont_thaw = generate_cont_thaw();
12801 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
12802 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
12803 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
12804 }
12805
12806 void generate_final_stubs() {
12807 // support for verify_oop (must happen after universe_init)
12808 if (VerifyOops) {
12809 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
12810 }
12811
12812 // arraycopy stubs used by compilers
12813 generate_arraycopy_stubs();
12814
12815 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
|