487 __ blr(c_rarg4);
488
489 // we do this here because the notify will already have been done
490 // if we get to the next instruction via an exception
491 //
492 // n.b. adding this instruction here affects the calculation of
493 // whether or not a routine returns to the call stub (used when
494 // doing stack walks) since the normal test is to check the return
495 // pc against the address saved below. so we may need to allow for
496 // this extra instruction in the check.
497
498 // save current address for use by exception handling code
499
500 return_address = __ pc();
501 entries.append(return_address);
502
503 // store result depending on type (everything that is not
504 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
505 // n.b. this assumes Java returns an integral result in r0
506 // and a floating result in j_farg0
507 __ ldr(j_rarg2, result);
508 Label is_long, is_float, is_double, exit;
509 __ ldr(j_rarg1, result_type);
510 __ cmp(j_rarg1, (u1)T_OBJECT);
511 __ br(Assembler::EQ, is_long);
512 __ cmp(j_rarg1, (u1)T_LONG);
513 __ br(Assembler::EQ, is_long);
514 __ cmp(j_rarg1, (u1)T_FLOAT);
515 __ br(Assembler::EQ, is_float);
516 __ cmp(j_rarg1, (u1)T_DOUBLE);
517 __ br(Assembler::EQ, is_double);
518
519 // handle T_INT case
520 __ strw(r0, Address(j_rarg2));
521
522 __ BIND(exit);
523
524 // pop parameters
525 __ sub(esp, rfp, -sp_after_call_off * wordSize);
526
527 #ifdef ASSERT
528 // verify that threads correspond
529 {
530 Label L, S;
531 __ ldr(rscratch1, thread);
532 __ cmp(rthread, rscratch1);
533 __ br(Assembler::NE, S);
534 __ get_thread(rscratch1);
535 __ cmp(rthread, rscratch1);
536 __ br(Assembler::EQ, L);
537 __ BIND(S);
538 __ stop("StubRoutines::call_stub: threads must correspond");
539 __ BIND(L);
540 }
552 __ ldp(r26, r25, r26_save);
553 __ ldp(r24, r23, r24_save);
554 __ ldp(r22, r21, r22_save);
555 __ ldp(r20, r19, r20_save);
556
557 // restore fpcr
558 __ ldr(rscratch1, fpcr_save);
559 __ set_fpcr(rscratch1);
560
561 __ ldp(c_rarg0, c_rarg1, call_wrapper);
562 __ ldrw(c_rarg2, result_type);
563 __ ldr(c_rarg3, method);
564 __ ldp(c_rarg4, c_rarg5, entry_point);
565 __ ldp(c_rarg6, c_rarg7, parameter_size);
566
567 // leave frame and return to caller
568 __ leave();
569 __ ret(lr);
570
571 // handle return types different from T_INT
572
573 __ BIND(is_long);
574 __ str(r0, Address(j_rarg2, 0));
575 __ br(Assembler::AL, exit);
576
577 __ BIND(is_float);
578 __ strs(j_farg0, Address(j_rarg2, 0));
579 __ br(Assembler::AL, exit);
580
581 __ BIND(is_double);
582 __ strd(j_farg0, Address(j_rarg2, 0));
583 __ br(Assembler::AL, exit);
584
585 // record the stub entry and end plus the auxiliary entry
586 store_archive_data(stub_id, start, __ pc(), &entries);
587
588 return start;
589 }
590
591 // Return point for a Java call if there's an exception thrown in
592 // Java code. The exception is caught and transformed into a
593 // pending exception stored in JavaThread that can be tested from
594 // within the VM.
595 //
596 // Note: Usually the parameters are removed by the callee. In case
597 // of an exception crossing an activation frame boundary, that is
598 // not the case if the callee is compiled code => need to setup the
599 // rsp.
600 //
601 // r0: exception oop
602
2590 // |array_tag| | header_size | element_type | |log2_element_size|
2591 // 32 30 24 16 8 2 0
2592 //
2593 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2594 //
2595
2596 const int lh_offset = in_bytes(Klass::layout_helper_offset());
2597
2598 // Handle objArrays completely differently...
2599 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2600 __ ldrw(lh, Address(scratch_src_klass, lh_offset));
2601 __ movw(rscratch1, objArray_lh);
2602 __ eorw(rscratch2, lh, rscratch1);
2603 __ cbzw(rscratch2, L_objArray);
2604
2605 // if (src->klass() != dst->klass()) return -1;
2606 __ load_klass(rscratch2, dst, rscratch1);
2607 __ eor(rscratch2, rscratch2, scratch_src_klass);
2608 __ cbnz(rscratch2, L_failed);
2609
2610 // if (!src->is_Array()) return -1;
2611 __ tbz(lh, 31, L_failed); // i.e. (lh >= 0)
2612
2613 // At this point, it is known to be a typeArray (array_tag 0x3).
2614 #ifdef ASSERT
2615 {
2616 BLOCK_COMMENT("assert primitive array {");
2617 Label L;
2618 __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2619 __ cmpw(lh, rscratch2);
2620 __ br(Assembler::GE, L);
2621 __ stop("must be a primitive array");
2622 __ bind(L);
2623 BLOCK_COMMENT("} assert primitive array done");
2624 }
2625 #endif
2626
2627 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2628 rscratch2, L_failed);
2629
12381 entries.append((address)aarch64_atomic_xchg_4_impl);
12382 entries.append((address)aarch64_atomic_xchg_8_impl);
12383 entries.append((address)aarch64_atomic_cmpxchg_1_impl);
12384 entries.append((address)aarch64_atomic_cmpxchg_4_impl);
12385 entries.append((address)aarch64_atomic_cmpxchg_8_impl);
12386 entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl);
12387 entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl);
12388 entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl);
12389 entries.append((address)aarch64_atomic_cmpxchg_4_release_impl);
12390 entries.append((address)aarch64_atomic_cmpxchg_8_release_impl);
12391 entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl);
12392 entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl);
12393
12394 assert(entries.length() == entry_count - 1,
12395 "unexpected extra entry count %d", entries.length());
12396
12397 store_archive_data(stub_id, start, end, &entries);
12398 }
12399 #endif // LINUX
12400
12401 address generate_cont_thaw(Continuation::thaw_kind kind) {
12402 bool return_barrier = Continuation::is_thaw_return_barrier(kind);
12403 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
12404
12405 address start = __ pc();
12406
12407 if (return_barrier) {
12408 __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
12409 __ mov(sp, rscratch1);
12410 }
12411 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
12412
12413 if (return_barrier) {
12414 // preserve possible return value from a method returning to the return barrier
12415 __ fmovd(rscratch1, v0);
12416 __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
12417 }
12418
12419 __ movw(c_rarg1, (return_barrier ? 1 : 0));
12420 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
12421 __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
12422
12423 if (return_barrier) {
12424 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
12425 __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
12426 __ fmovd(v0, rscratch1);
12427 }
12428 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
12429
12430
12431 Label thaw_success;
12432 // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
12433 __ cbnz(rscratch2, thaw_success);
12434 __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
12435 __ br(rscratch1);
12436 __ bind(thaw_success);
12437
12438 // make room for the thawed frames
12439 __ sub(rscratch1, sp, rscratch2);
12440 __ andr(rscratch1, rscratch1, -16); // align
12441 __ mov(sp, rscratch1);
12442
12443 if (return_barrier) {
12444 // save original return value -- again
12445 __ fmovd(rscratch1, v0);
12446 __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
12447 }
12448
12449 // If we want, we can templatize thaw by kind, and have three different entries
12450 __ movw(c_rarg1, (uint32_t)kind);
12451
12452 __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
12453 __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
12454
12455 if (return_barrier) {
12456 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
12457 __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
12458 __ fmovd(v0, rscratch1);
12459 } else {
12460 __ mov(r0, zr); // return 0 (success) from doYield
12461 }
12462
12463 // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
12464 __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
12465 __ mov(rfp, sp);
12466
12467 if (return_barrier_exception) {
12468 __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
12469 __ authenticate_return_address(c_rarg1);
12470 __ verify_oop(r0);
12471 // save return value containing the exception oop in callee-saved R19
12472 __ mov(r19, r0);
12473
12474 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
12475
12476 // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
12477 // __ reinitialize_ptrue();
12478
|
487 __ blr(c_rarg4);
488
489 // we do this here because the notify will already have been done
490 // if we get to the next instruction via an exception
491 //
492 // n.b. adding this instruction here affects the calculation of
493 // whether or not a routine returns to the call stub (used when
494 // doing stack walks) since the normal test is to check the return
495 // pc against the address saved below. so we may need to allow for
496 // this extra instruction in the check.
497
498 // save current address for use by exception handling code
499
500 return_address = __ pc();
501 entries.append(return_address);
502
503 // store result depending on type (everything that is not
504 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
505 // n.b. this assumes Java returns an integral result in r0
506 // and a floating result in j_farg0
507 // All of j_rargN may be used to return inline type fields so be careful
508 // not to clobber those.
509 // SharedRuntime::generate_buffered_inline_type_adapter() knows the register
510 // assignment of Rresult below.
511 Register Rresult = r14, Rresult_type = r15;
512 __ ldr(Rresult, result);
513 Label is_long, is_float, is_double, check_prim, exit;
514 __ ldr(Rresult_type, result_type);
515 __ cmp(Rresult_type, (u1)T_OBJECT);
516 __ br(Assembler::EQ, check_prim);
517 __ cmp(Rresult_type, (u1)T_LONG);
518 __ br(Assembler::EQ, is_long);
519 __ cmp(Rresult_type, (u1)T_FLOAT);
520 __ br(Assembler::EQ, is_float);
521 __ cmp(Rresult_type, (u1)T_DOUBLE);
522 __ br(Assembler::EQ, is_double);
523
524 // handle T_INT case
525 __ strw(r0, Address(Rresult));
526
527 __ BIND(exit);
528
529 // pop parameters
530 __ sub(esp, rfp, -sp_after_call_off * wordSize);
531
532 #ifdef ASSERT
533 // verify that threads correspond
534 {
535 Label L, S;
536 __ ldr(rscratch1, thread);
537 __ cmp(rthread, rscratch1);
538 __ br(Assembler::NE, S);
539 __ get_thread(rscratch1);
540 __ cmp(rthread, rscratch1);
541 __ br(Assembler::EQ, L);
542 __ BIND(S);
543 __ stop("StubRoutines::call_stub: threads must correspond");
544 __ BIND(L);
545 }
557 __ ldp(r26, r25, r26_save);
558 __ ldp(r24, r23, r24_save);
559 __ ldp(r22, r21, r22_save);
560 __ ldp(r20, r19, r20_save);
561
562 // restore fpcr
563 __ ldr(rscratch1, fpcr_save);
564 __ set_fpcr(rscratch1);
565
566 __ ldp(c_rarg0, c_rarg1, call_wrapper);
567 __ ldrw(c_rarg2, result_type);
568 __ ldr(c_rarg3, method);
569 __ ldp(c_rarg4, c_rarg5, entry_point);
570 __ ldp(c_rarg6, c_rarg7, parameter_size);
571
572 // leave frame and return to caller
573 __ leave();
574 __ ret(lr);
575
576 // handle return types different from T_INT
577 __ BIND(check_prim);
578 if (InlineTypeReturnedAsFields) {
579 // Check for scalarized return value
580 __ tbz(r0, 0, is_long);
581 // Load pack handler address
582 __ andr(rscratch1, r0, -2);
583 __ ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
584 __ ldr(rscratch1, Address(rscratch1, InlineKlass::pack_handler_jobject_offset()));
585 __ blr(rscratch1);
586 __ b(exit);
587 }
588
589 __ BIND(is_long);
590 __ str(r0, Address(Rresult, 0));
591 __ br(Assembler::AL, exit);
592
593 __ BIND(is_float);
594 __ strs(j_farg0, Address(Rresult, 0));
595 __ br(Assembler::AL, exit);
596
597 __ BIND(is_double);
598 __ strd(j_farg0, Address(Rresult, 0));
599 __ br(Assembler::AL, exit);
600
601 // record the stub entry and end plus the auxiliary entry
602 store_archive_data(stub_id, start, __ pc(), &entries);
603
604 return start;
605 }
606
607 // Return point for a Java call if there's an exception thrown in
608 // Java code. The exception is caught and transformed into a
609 // pending exception stored in JavaThread that can be tested from
610 // within the VM.
611 //
612 // Note: Usually the parameters are removed by the callee. In case
613 // of an exception crossing an activation frame boundary, that is
614 // not the case if the callee is compiled code => need to setup the
615 // rsp.
616 //
617 // r0: exception oop
618
2606 // |array_tag| | header_size | element_type | |log2_element_size|
2607 // 32 30 24 16 8 2 0
2608 //
2609 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2610 //
2611
2612 const int lh_offset = in_bytes(Klass::layout_helper_offset());
2613
2614 // Handle objArrays completely differently...
2615 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2616 __ ldrw(lh, Address(scratch_src_klass, lh_offset));
2617 __ movw(rscratch1, objArray_lh);
2618 __ eorw(rscratch2, lh, rscratch1);
2619 __ cbzw(rscratch2, L_objArray);
2620
2621 // if (src->klass() != dst->klass()) return -1;
2622 __ load_klass(rscratch2, dst, rscratch1);
2623 __ eor(rscratch2, rscratch2, scratch_src_klass);
2624 __ cbnz(rscratch2, L_failed);
2625
2626 // Check for flat inline type array -> return -1
2627 __ test_flat_array_oop(src, rscratch2, L_failed);
2628
2629 // Check for null-free (non-flat) inline type array -> handle as object array
2630 __ test_null_free_array_oop(src, rscratch2, L_objArray);
2631
2632 // if (!src->is_Array()) return -1;
2633 __ tbz(lh, 31, L_failed); // i.e. (lh >= 0)
2634
2635 // At this point, it is known to be a typeArray (array_tag 0x3).
2636 #ifdef ASSERT
2637 {
2638 BLOCK_COMMENT("assert primitive array {");
2639 Label L;
2640 __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2641 __ cmpw(lh, rscratch2);
2642 __ br(Assembler::GE, L);
2643 __ stop("must be a primitive array");
2644 __ bind(L);
2645 BLOCK_COMMENT("} assert primitive array done");
2646 }
2647 #endif
2648
2649 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2650 rscratch2, L_failed);
2651
12403 entries.append((address)aarch64_atomic_xchg_4_impl);
12404 entries.append((address)aarch64_atomic_xchg_8_impl);
12405 entries.append((address)aarch64_atomic_cmpxchg_1_impl);
12406 entries.append((address)aarch64_atomic_cmpxchg_4_impl);
12407 entries.append((address)aarch64_atomic_cmpxchg_8_impl);
12408 entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl);
12409 entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl);
12410 entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl);
12411 entries.append((address)aarch64_atomic_cmpxchg_4_release_impl);
12412 entries.append((address)aarch64_atomic_cmpxchg_8_release_impl);
12413 entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl);
12414 entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl);
12415
12416 assert(entries.length() == entry_count - 1,
12417 "unexpected extra entry count %d", entries.length());
12418
12419 store_archive_data(stub_id, start, end, &entries);
12420 }
12421 #endif // LINUX
12422
12423 static void save_return_registers(MacroAssembler* masm) {
12424 if (InlineTypeReturnedAsFields) {
12425 masm->push(RegSet::range(r0, r7), sp);
12426 masm->sub(sp, sp, 4 * wordSize);
12427 masm->st1(v0, v1, v2, v3, masm->T1D, Address(sp));
12428 masm->sub(sp, sp, 4 * wordSize);
12429 masm->st1(v4, v5, v6, v7, masm->T1D, Address(sp));
12430 } else {
12431 masm->fmovd(rscratch1, v0);
12432 masm->stp(rscratch1, r0, Address(masm->pre(sp, -2 * wordSize)));
12433 }
12434 }
12435
12436 static void restore_return_registers(MacroAssembler* masm) {
12437 if (InlineTypeReturnedAsFields) {
12438 masm->ld1(v4, v5, v6, v7, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
12439 masm->ld1(v0, v1, v2, v3, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
12440 masm->pop(RegSet::range(r0, r7), sp);
12441 } else {
12442 masm->ldp(rscratch1, r0, Address(masm->post(sp, 2 * wordSize)));
12443 masm->fmovd(v0, rscratch1);
12444 }
12445 }
12446
12447 address generate_cont_thaw(Continuation::thaw_kind kind) {
12448 bool return_barrier = Continuation::is_thaw_return_barrier(kind);
12449 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
12450
12451 address start = __ pc();
12452
12453 if (return_barrier) {
12454 __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
12455 __ mov(sp, rscratch1);
12456 }
12457 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
12458
12459 if (return_barrier) {
12460 // preserve possible return value from a method returning to the return barrier
12461 save_return_registers(_masm);
12462 }
12463
12464 __ movw(c_rarg1, (return_barrier ? 1 : 0));
12465 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
12466 __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
12467
12468 if (return_barrier) {
12469 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
12470 restore_return_registers(_masm);
12471 }
12472 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
12473
12474
12475 Label thaw_success;
12476 // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
12477 __ cbnz(rscratch2, thaw_success);
12478 __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
12479 __ br(rscratch1);
12480 __ bind(thaw_success);
12481
12482 // make room for the thawed frames
12483 __ sub(rscratch1, sp, rscratch2);
12484 __ andr(rscratch1, rscratch1, -16); // align
12485 __ mov(sp, rscratch1);
12486
12487 if (return_barrier) {
12488 // save original return value -- again
12489 save_return_registers(_masm);
12490 }
12491
12492 // If we want, we can templatize thaw by kind, and have three different entries
12493 __ movw(c_rarg1, (uint32_t)kind);
12494
12495 __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
12496 __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
12497
12498 if (return_barrier) {
12499 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
12500 restore_return_registers(_masm);
12501 } else {
12502 __ mov(r0, zr); // return 0 (success) from doYield
12503 }
12504
12505 // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
12506 __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
12507 __ mov(rfp, sp);
12508
12509 if (return_barrier_exception) {
12510 __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
12511 __ authenticate_return_address(c_rarg1);
12512 __ verify_oop(r0);
12513 // save return value containing the exception oop in callee-saved R19
12514 __ mov(r19, r0);
12515
12516 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
12517
12518 // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
12519 // __ reinitialize_ptrue();
12520
|