311 __ mov(r19_sender_sp, sp);
312 __ blr(c_rarg4);
313
314 // we do this here because the notify will already have been done
315 // if we get to the next instruction via an exception
316 //
317 // n.b. adding this instruction here affects the calculation of
318 // whether or not a routine returns to the call stub (used when
319 // doing stack walks) since the normal test is to check the return
320 // pc against the address saved below. so we may need to allow for
321 // this extra instruction in the check.
322
323 // save current address for use by exception handling code
324
325 return_address = __ pc();
326
327 // store result depending on type (everything that is not
328 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
329 // n.b. this assumes Java returns an integral result in r0
330 // and a floating result in j_farg0
331 __ ldr(j_rarg2, result);
332 Label is_long, is_float, is_double, exit;
333 __ ldr(j_rarg1, result_type);
334 __ cmp(j_rarg1, (u1)T_OBJECT);
335 __ br(Assembler::EQ, is_long);
336 __ cmp(j_rarg1, (u1)T_LONG);
337 __ br(Assembler::EQ, is_long);
338 __ cmp(j_rarg1, (u1)T_FLOAT);
339 __ br(Assembler::EQ, is_float);
340 __ cmp(j_rarg1, (u1)T_DOUBLE);
341 __ br(Assembler::EQ, is_double);
342
343 // handle T_INT case
344 __ strw(r0, Address(j_rarg2));
345
346 __ BIND(exit);
347
348 // pop parameters
349 __ sub(esp, rfp, -sp_after_call_off * wordSize);
350
351 #ifdef ASSERT
352 // verify that threads correspond
353 {
354 Label L, S;
355 __ ldr(rscratch1, thread);
356 __ cmp(rthread, rscratch1);
357 __ br(Assembler::NE, S);
358 __ get_thread(rscratch1);
359 __ cmp(rthread, rscratch1);
360 __ br(Assembler::EQ, L);
361 __ BIND(S);
362 __ stop("StubRoutines::call_stub: threads must correspond");
363 __ BIND(L);
364 }
376 __ ldp(r26, r25, r26_save);
377 __ ldp(r24, r23, r24_save);
378 __ ldp(r22, r21, r22_save);
379 __ ldp(r20, r19, r20_save);
380
381 // restore fpcr
382 __ ldr(rscratch1, fpcr_save);
383 __ set_fpcr(rscratch1);
384
385 __ ldp(c_rarg0, c_rarg1, call_wrapper);
386 __ ldrw(c_rarg2, result_type);
387 __ ldr(c_rarg3, method);
388 __ ldp(c_rarg4, c_rarg5, entry_point);
389 __ ldp(c_rarg6, c_rarg7, parameter_size);
390
391 // leave frame and return to caller
392 __ leave();
393 __ ret(lr);
394
395 // handle return types different from T_INT
396
397 __ BIND(is_long);
398 __ str(r0, Address(j_rarg2, 0));
399 __ br(Assembler::AL, exit);
400
401 __ BIND(is_float);
402 __ strs(j_farg0, Address(j_rarg2, 0));
403 __ br(Assembler::AL, exit);
404
405 __ BIND(is_double);
406 __ strd(j_farg0, Address(j_rarg2, 0));
407 __ br(Assembler::AL, exit);
408
409 return start;
410 }
411
412 // Return point for a Java call if there's an exception thrown in
413 // Java code. The exception is caught and transformed into a
414 // pending exception stored in JavaThread that can be tested from
415 // within the VM.
416 //
417 // Note: Usually the parameters are removed by the callee. In case
418 // of an exception crossing an activation frame boundary, that is
419 // not the case if the callee is compiled code => need to setup the
420 // rsp.
421 //
422 // r0: exception oop
423
424 address generate_catch_exception() {
425 StubId stub_id = StubId::stubgen_catch_exception_id;
426 StubCodeMark mark(this, stub_id);
2208 // |array_tag| | header_size | element_type | |log2_element_size|
2209 // 32 30 24 16 8 2 0
2210 //
2211 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2212 //
2213
2214 const int lh_offset = in_bytes(Klass::layout_helper_offset());
2215
2216 // Handle objArrays completely differently...
2217 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2218 __ ldrw(lh, Address(scratch_src_klass, lh_offset));
2219 __ movw(rscratch1, objArray_lh);
2220 __ eorw(rscratch2, lh, rscratch1);
2221 __ cbzw(rscratch2, L_objArray);
2222
2223 // if (src->klass() != dst->klass()) return -1;
2224 __ load_klass(rscratch2, dst);
2225 __ eor(rscratch2, rscratch2, scratch_src_klass);
2226 __ cbnz(rscratch2, L_failed);
2227
2228 // if (!src->is_Array()) return -1;
2229 __ tbz(lh, 31, L_failed); // i.e. (lh >= 0)
2230
2231 // At this point, it is known to be a typeArray (array_tag 0x3).
2232 #ifdef ASSERT
2233 {
2234 BLOCK_COMMENT("assert primitive array {");
2235 Label L;
2236 __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2237 __ cmpw(lh, rscratch2);
2238 __ br(Assembler::GE, L);
2239 __ stop("must be a primitive array");
2240 __ bind(L);
2241 BLOCK_COMMENT("} assert primitive array done");
2242 }
2243 #endif
2244
2245 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2246 rscratch2, L_failed);
2247
10506 gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
10507
10508 AtomicStubMark mark_cmpxchg_4_release
10509 (_masm, &aarch64_atomic_cmpxchg_4_release_impl);
10510 gen_cas_entry(MacroAssembler::word, memory_order_release);
10511 AtomicStubMark mark_cmpxchg_8_release
10512 (_masm, &aarch64_atomic_cmpxchg_8_release_impl);
10513 gen_cas_entry(MacroAssembler::xword, memory_order_release);
10514
10515 AtomicStubMark mark_cmpxchg_4_seq_cst
10516 (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
10517 gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
10518 AtomicStubMark mark_cmpxchg_8_seq_cst
10519 (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
10520 gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
10521
10522 ICache::invalidate_range(first_entry, __ pc() - first_entry);
10523 }
10524 #endif // LINUX
10525
10526 address generate_cont_thaw(Continuation::thaw_kind kind) {
10527 bool return_barrier = Continuation::is_thaw_return_barrier(kind);
10528 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
10529
10530 address start = __ pc();
10531
10532 if (return_barrier) {
10533 __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
10534 __ mov(sp, rscratch1);
10535 }
10536 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
10537
10538 if (return_barrier) {
10539 // preserve possible return value from a method returning to the return barrier
10540 __ fmovd(rscratch1, v0);
10541 __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
10542 }
10543
10544 __ movw(c_rarg1, (return_barrier ? 1 : 0));
10545 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
10546 __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
10547
10548 if (return_barrier) {
10549 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
10550 __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
10551 __ fmovd(v0, rscratch1);
10552 }
10553 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
10554
10555
10556 Label thaw_success;
10557 // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
10558 __ cbnz(rscratch2, thaw_success);
10559 __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
10560 __ br(rscratch1);
10561 __ bind(thaw_success);
10562
10563 // make room for the thawed frames
10564 __ sub(rscratch1, sp, rscratch2);
10565 __ andr(rscratch1, rscratch1, -16); // align
10566 __ mov(sp, rscratch1);
10567
10568 if (return_barrier) {
10569 // save original return value -- again
10570 __ fmovd(rscratch1, v0);
10571 __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
10572 }
10573
10574 // If we want, we can templatize thaw by kind, and have three different entries
10575 __ movw(c_rarg1, (uint32_t)kind);
10576
10577 __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
10578 __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
10579
10580 if (return_barrier) {
10581 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
10582 __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
10583 __ fmovd(v0, rscratch1);
10584 } else {
10585 __ mov(r0, zr); // return 0 (success) from doYield
10586 }
10587
10588 // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
10589 __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
10590 __ mov(rfp, sp);
10591
10592 if (return_barrier_exception) {
10593 __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
10594 __ authenticate_return_address(c_rarg1);
10595 __ verify_oop(r0);
10596 // save return value containing the exception oop in callee-saved R19
10597 __ mov(r19, r0);
10598
10599 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
10600
10601 // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
10602 // __ reinitialize_ptrue();
10603
11700 // assert(Ra == Pa_base[j], "must be");
11701 // MACC(Ra, Ra, t0, t1, t2);
11702 // }
11703 // iters = (2*len-i)/2;
11704 // assert(iters == len-j, "must be");
11705 // for (; iters--; j++) {
11706 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
11707 // MACC(Rm, Rn, t0, t1, t2);
11708 // Rm = *++Pm;
11709 // Rn = *--Pn;
11710 // }
11711 // Pm_base[i-len] = t0;
11712 // t0 = t1; t1 = t2; t2 = 0;
11713 // }
11714
11715 // while (t0)
11716 // t0 = sub(Pm_base, Pn_base, t0, len);
11717 // }
11718 };
11719
11720 // Initialization
11721 void generate_preuniverse_stubs() {
11722 // preuniverse stubs are not needed for aarch64
11723 }
11724
11725 void generate_initial_stubs() {
11726 // Generate initial stubs and initializes the entry points
11727
11728 // entry points that exist in all platforms Note: This is code
11729 // that could be shared among different platforms - however the
11730 // benefit seems to be smaller than the disadvantage of having a
11731 // much more complicated generator structure. See also comment in
11732 // stubRoutines.hpp.
11733
11734 StubRoutines::_forward_exception_entry = generate_forward_exception();
11735
11736 StubRoutines::_call_stub_entry =
11737 generate_call_stub(StubRoutines::_call_stub_return_address);
11738
11739 // is referenced by megamorphic call
11748 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
11749 }
11750
11751 if (UseCRC32CIntrinsics) {
11752 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
11753 }
11754
11755 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
11756 StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
11757 }
11758
11759 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
11760 StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
11761 }
11762
11763 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
11764 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
11765 StubRoutines::_hf2f = generate_float16ToFloat();
11766 StubRoutines::_f2hf = generate_floatToFloat16();
11767 }
11768 }
11769
11770 void generate_continuation_stubs() {
11771 // Continuation stubs:
11772 StubRoutines::_cont_thaw = generate_cont_thaw();
11773 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
11774 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
11775 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
11776 }
11777
11778 void generate_final_stubs() {
11779 // support for verify_oop (must happen after universe_init)
11780 if (VerifyOops) {
11781 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
11782 }
11783
11784 // arraycopy stubs used by compilers
11785 generate_arraycopy_stubs();
11786
11787 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
|
311 __ mov(r19_sender_sp, sp);
312 __ blr(c_rarg4);
313
314 // we do this here because the notify will already have been done
315 // if we get to the next instruction via an exception
316 //
317 // n.b. adding this instruction here affects the calculation of
318 // whether or not a routine returns to the call stub (used when
319 // doing stack walks) since the normal test is to check the return
320 // pc against the address saved below. so we may need to allow for
321 // this extra instruction in the check.
322
323 // save current address for use by exception handling code
324
325 return_address = __ pc();
326
327 // store result depending on type (everything that is not
328 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
329 // n.b. this assumes Java returns an integral result in r0
330 // and a floating result in j_farg0
331 // All of j_rargN may be used to return inline type fields so be careful
332 // not to clobber those.
333 // SharedRuntime::generate_buffered_inline_type_adapter() knows the register
334 // assignment of Rresult below.
335 Register Rresult = r14, Rresult_type = r15;
336 __ ldr(Rresult, result);
337 Label is_long, is_float, is_double, check_prim, exit;
338 __ ldr(Rresult_type, result_type);
339 __ cmp(Rresult_type, (u1)T_OBJECT);
340 __ br(Assembler::EQ, check_prim);
341 __ cmp(Rresult_type, (u1)T_LONG);
342 __ br(Assembler::EQ, is_long);
343 __ cmp(Rresult_type, (u1)T_FLOAT);
344 __ br(Assembler::EQ, is_float);
345 __ cmp(Rresult_type, (u1)T_DOUBLE);
346 __ br(Assembler::EQ, is_double);
347
348 // handle T_INT case
349 __ strw(r0, Address(Rresult));
350
351 __ BIND(exit);
352
353 // pop parameters
354 __ sub(esp, rfp, -sp_after_call_off * wordSize);
355
356 #ifdef ASSERT
357 // verify that threads correspond
358 {
359 Label L, S;
360 __ ldr(rscratch1, thread);
361 __ cmp(rthread, rscratch1);
362 __ br(Assembler::NE, S);
363 __ get_thread(rscratch1);
364 __ cmp(rthread, rscratch1);
365 __ br(Assembler::EQ, L);
366 __ BIND(S);
367 __ stop("StubRoutines::call_stub: threads must correspond");
368 __ BIND(L);
369 }
381 __ ldp(r26, r25, r26_save);
382 __ ldp(r24, r23, r24_save);
383 __ ldp(r22, r21, r22_save);
384 __ ldp(r20, r19, r20_save);
385
386 // restore fpcr
387 __ ldr(rscratch1, fpcr_save);
388 __ set_fpcr(rscratch1);
389
390 __ ldp(c_rarg0, c_rarg1, call_wrapper);
391 __ ldrw(c_rarg2, result_type);
392 __ ldr(c_rarg3, method);
393 __ ldp(c_rarg4, c_rarg5, entry_point);
394 __ ldp(c_rarg6, c_rarg7, parameter_size);
395
396 // leave frame and return to caller
397 __ leave();
398 __ ret(lr);
399
400 // handle return types different from T_INT
401 __ BIND(check_prim);
402 if (InlineTypeReturnedAsFields) {
403 // Check for scalarized return value
404 __ tbz(r0, 0, is_long);
405 // Load pack handler address
406 __ andr(rscratch1, r0, -2);
407 __ ldr(rscratch1, Address(rscratch1, InstanceKlass::adr_inlineklass_fixed_block_offset()));
408 __ ldr(rscratch1, Address(rscratch1, InlineKlass::pack_handler_jobject_offset()));
409 __ blr(rscratch1);
410 __ b(exit);
411 }
412
413 __ BIND(is_long);
414 __ str(r0, Address(Rresult, 0));
415 __ br(Assembler::AL, exit);
416
417 __ BIND(is_float);
418 __ strs(j_farg0, Address(Rresult, 0));
419 __ br(Assembler::AL, exit);
420
421 __ BIND(is_double);
422 __ strd(j_farg0, Address(Rresult, 0));
423 __ br(Assembler::AL, exit);
424
425 return start;
426 }
427
428 // Return point for a Java call if there's an exception thrown in
429 // Java code. The exception is caught and transformed into a
430 // pending exception stored in JavaThread that can be tested from
431 // within the VM.
432 //
433 // Note: Usually the parameters are removed by the callee. In case
434 // of an exception crossing an activation frame boundary, that is
435 // not the case if the callee is compiled code => need to setup the
436 // rsp.
437 //
438 // r0: exception oop
439
440 address generate_catch_exception() {
441 StubId stub_id = StubId::stubgen_catch_exception_id;
442 StubCodeMark mark(this, stub_id);
2224 // |array_tag| | header_size | element_type | |log2_element_size|
2225 // 32 30 24 16 8 2 0
2226 //
2227 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2228 //
2229
2230 const int lh_offset = in_bytes(Klass::layout_helper_offset());
2231
2232 // Handle objArrays completely differently...
2233 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2234 __ ldrw(lh, Address(scratch_src_klass, lh_offset));
2235 __ movw(rscratch1, objArray_lh);
2236 __ eorw(rscratch2, lh, rscratch1);
2237 __ cbzw(rscratch2, L_objArray);
2238
2239 // if (src->klass() != dst->klass()) return -1;
2240 __ load_klass(rscratch2, dst);
2241 __ eor(rscratch2, rscratch2, scratch_src_klass);
2242 __ cbnz(rscratch2, L_failed);
2243
2244 // Check for flat inline type array -> return -1
2245 __ test_flat_array_oop(src, rscratch2, L_failed);
2246
2247 // Check for null-free (non-flat) inline type array -> handle as object array
2248 __ test_null_free_array_oop(src, rscratch2, L_objArray);
2249
2250 // if (!src->is_Array()) return -1;
2251 __ tbz(lh, 31, L_failed); // i.e. (lh >= 0)
2252
2253 // At this point, it is known to be a typeArray (array_tag 0x3).
2254 #ifdef ASSERT
2255 {
2256 BLOCK_COMMENT("assert primitive array {");
2257 Label L;
2258 __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2259 __ cmpw(lh, rscratch2);
2260 __ br(Assembler::GE, L);
2261 __ stop("must be a primitive array");
2262 __ bind(L);
2263 BLOCK_COMMENT("} assert primitive array done");
2264 }
2265 #endif
2266
2267 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2268 rscratch2, L_failed);
2269
10528 gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
10529
10530 AtomicStubMark mark_cmpxchg_4_release
10531 (_masm, &aarch64_atomic_cmpxchg_4_release_impl);
10532 gen_cas_entry(MacroAssembler::word, memory_order_release);
10533 AtomicStubMark mark_cmpxchg_8_release
10534 (_masm, &aarch64_atomic_cmpxchg_8_release_impl);
10535 gen_cas_entry(MacroAssembler::xword, memory_order_release);
10536
10537 AtomicStubMark mark_cmpxchg_4_seq_cst
10538 (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
10539 gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
10540 AtomicStubMark mark_cmpxchg_8_seq_cst
10541 (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
10542 gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
10543
10544 ICache::invalidate_range(first_entry, __ pc() - first_entry);
10545 }
10546 #endif // LINUX
10547
10548 static void save_return_registers(MacroAssembler* masm) {
10549 if (InlineTypeReturnedAsFields) {
10550 masm->push(RegSet::range(r0, r7), sp);
10551 masm->sub(sp, sp, 4 * wordSize);
10552 masm->st1(v0, v1, v2, v3, masm->T1D, Address(sp));
10553 masm->sub(sp, sp, 4 * wordSize);
10554 masm->st1(v4, v5, v6, v7, masm->T1D, Address(sp));
10555 } else {
10556 masm->fmovd(rscratch1, v0);
10557 masm->stp(rscratch1, r0, Address(masm->pre(sp, -2 * wordSize)));
10558 }
10559 }
10560
10561 static void restore_return_registers(MacroAssembler* masm) {
10562 if (InlineTypeReturnedAsFields) {
10563 masm->ld1(v4, v5, v6, v7, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
10564 masm->ld1(v0, v1, v2, v3, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
10565 masm->pop(RegSet::range(r0, r7), sp);
10566 } else {
10567 masm->ldp(rscratch1, r0, Address(masm->post(sp, 2 * wordSize)));
10568 masm->fmovd(v0, rscratch1);
10569 }
10570 }
10571
10572 address generate_cont_thaw(Continuation::thaw_kind kind) {
10573 bool return_barrier = Continuation::is_thaw_return_barrier(kind);
10574 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
10575
10576 address start = __ pc();
10577
10578 if (return_barrier) {
10579 __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
10580 __ mov(sp, rscratch1);
10581 }
10582 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
10583
10584 if (return_barrier) {
10585 // preserve possible return value from a method returning to the return barrier
10586 save_return_registers(_masm);
10587 }
10588
10589 __ movw(c_rarg1, (return_barrier ? 1 : 0));
10590 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
10591 __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
10592
10593 if (return_barrier) {
10594 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
10595 restore_return_registers(_masm);
10596 }
10597 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
10598
10599
10600 Label thaw_success;
10601 // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
10602 __ cbnz(rscratch2, thaw_success);
10603 __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
10604 __ br(rscratch1);
10605 __ bind(thaw_success);
10606
10607 // make room for the thawed frames
10608 __ sub(rscratch1, sp, rscratch2);
10609 __ andr(rscratch1, rscratch1, -16); // align
10610 __ mov(sp, rscratch1);
10611
10612 if (return_barrier) {
10613 // save original return value -- again
10614 save_return_registers(_masm);
10615 }
10616
10617 // If we want, we can templatize thaw by kind, and have three different entries
10618 __ movw(c_rarg1, (uint32_t)kind);
10619
10620 __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
10621 __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
10622
10623 if (return_barrier) {
10624 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
10625 restore_return_registers(_masm);
10626 } else {
10627 __ mov(r0, zr); // return 0 (success) from doYield
10628 }
10629
10630 // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
10631 __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
10632 __ mov(rfp, sp);
10633
10634 if (return_barrier_exception) {
10635 __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
10636 __ authenticate_return_address(c_rarg1);
10637 __ verify_oop(r0);
10638 // save return value containing the exception oop in callee-saved R19
10639 __ mov(r19, r0);
10640
10641 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
10642
10643 // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
10644 // __ reinitialize_ptrue();
10645
11742 // assert(Ra == Pa_base[j], "must be");
11743 // MACC(Ra, Ra, t0, t1, t2);
11744 // }
11745 // iters = (2*len-i)/2;
11746 // assert(iters == len-j, "must be");
11747 // for (; iters--; j++) {
11748 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
11749 // MACC(Rm, Rn, t0, t1, t2);
11750 // Rm = *++Pm;
11751 // Rn = *--Pn;
11752 // }
11753 // Pm_base[i-len] = t0;
11754 // t0 = t1; t1 = t2; t2 = 0;
11755 // }
11756
11757 // while (t0)
11758 // t0 = sub(Pm_base, Pn_base, t0, len);
11759 // }
11760 };
11761
11762 // Call here from the interpreter or compiled code to either load
11763 // multiple returned values from the inline type instance being
11764 // returned to registers or to store returned values to a newly
11765 // allocated inline type instance.
11766 address generate_return_value_stub(address destination, const char* name, bool has_res) {
11767 // We need to save all registers the calling convention may use so
11768 // the runtime calls read or update those registers. This needs to
11769 // be in sync with SharedRuntime::java_return_convention().
11770 // n.b. aarch64 asserts that frame::arg_reg_save_area_bytes == 0
11771 enum layout {
11772 j_rarg7_off = 0, j_rarg7_2, // j_rarg7 is r0
11773 j_rarg6_off, j_rarg6_2,
11774 j_rarg5_off, j_rarg5_2,
11775 j_rarg4_off, j_rarg4_2,
11776 j_rarg3_off, j_rarg3_2,
11777 j_rarg2_off, j_rarg2_2,
11778 j_rarg1_off, j_rarg1_2,
11779 j_rarg0_off, j_rarg0_2,
11780
11781 j_farg7_off, j_farg7_2,
11782 j_farg6_off, j_farg6_2,
11783 j_farg5_off, j_farg5_2,
11784 j_farg4_off, j_farg4_2,
11785 j_farg3_off, j_farg3_2,
11786 j_farg2_off, j_farg2_2,
11787 j_farg1_off, j_farg1_2,
11788 j_farg0_off, j_farg0_2,
11789
11790 rfp_off, rfp_off2,
11791 return_off, return_off2,
11792
11793 framesize // inclusive of return address
11794 };
11795
11796 CodeBuffer code(name, 512, 64);
11797 MacroAssembler* masm = new MacroAssembler(&code);
11798
11799 int frame_size_in_bytes = align_up(framesize*BytesPerInt, 16);
11800 assert(frame_size_in_bytes == framesize*BytesPerInt, "misaligned");
11801 int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
11802 int frame_size_in_words = frame_size_in_bytes / wordSize;
11803
11804 OopMapSet* oop_maps = new OopMapSet();
11805 OopMap* map = new OopMap(frame_size_in_slots, 0);
11806
11807 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg7_off), j_rarg7->as_VMReg());
11808 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg6_off), j_rarg6->as_VMReg());
11809 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg5_off), j_rarg5->as_VMReg());
11810 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg4_off), j_rarg4->as_VMReg());
11811 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg3_off), j_rarg3->as_VMReg());
11812 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg2_off), j_rarg2->as_VMReg());
11813 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg1_off), j_rarg1->as_VMReg());
11814 map->set_callee_saved(VMRegImpl::stack2reg(j_rarg0_off), j_rarg0->as_VMReg());
11815
11816 map->set_callee_saved(VMRegImpl::stack2reg(j_farg0_off), j_farg0->as_VMReg());
11817 map->set_callee_saved(VMRegImpl::stack2reg(j_farg1_off), j_farg1->as_VMReg());
11818 map->set_callee_saved(VMRegImpl::stack2reg(j_farg2_off), j_farg2->as_VMReg());
11819 map->set_callee_saved(VMRegImpl::stack2reg(j_farg3_off), j_farg3->as_VMReg());
11820 map->set_callee_saved(VMRegImpl::stack2reg(j_farg4_off), j_farg4->as_VMReg());
11821 map->set_callee_saved(VMRegImpl::stack2reg(j_farg5_off), j_farg5->as_VMReg());
11822 map->set_callee_saved(VMRegImpl::stack2reg(j_farg6_off), j_farg6->as_VMReg());
11823 map->set_callee_saved(VMRegImpl::stack2reg(j_farg7_off), j_farg7->as_VMReg());
11824
11825 address start = __ pc();
11826
11827 __ enter(); // Save FP and LR before call
11828
11829 __ stpd(j_farg1, j_farg0, Address(__ pre(sp, -2 * wordSize)));
11830 __ stpd(j_farg3, j_farg2, Address(__ pre(sp, -2 * wordSize)));
11831 __ stpd(j_farg5, j_farg4, Address(__ pre(sp, -2 * wordSize)));
11832 __ stpd(j_farg7, j_farg6, Address(__ pre(sp, -2 * wordSize)));
11833
11834 __ stp(j_rarg1, j_rarg0, Address(__ pre(sp, -2 * wordSize)));
11835 __ stp(j_rarg3, j_rarg2, Address(__ pre(sp, -2 * wordSize)));
11836 __ stp(j_rarg5, j_rarg4, Address(__ pre(sp, -2 * wordSize)));
11837 __ stp(j_rarg7, j_rarg6, Address(__ pre(sp, -2 * wordSize)));
11838
11839 int frame_complete = __ offset();
11840
11841 // Set up last_Java_sp and last_Java_fp
11842 address the_pc = __ pc();
11843 __ set_last_Java_frame(sp, noreg, the_pc, rscratch1);
11844
11845 // Call runtime
11846 __ mov(c_rarg1, r0);
11847 __ mov(c_rarg0, rthread);
11848
11849 __ mov(rscratch1, destination);
11850 __ blr(rscratch1);
11851
11852 oop_maps->add_gc_map(the_pc - start, map);
11853
11854 __ reset_last_Java_frame(false);
11855
11856 __ ldp(j_rarg7, j_rarg6, Address(__ post(sp, 2 * wordSize)));
11857 __ ldp(j_rarg5, j_rarg4, Address(__ post(sp, 2 * wordSize)));
11858 __ ldp(j_rarg3, j_rarg2, Address(__ post(sp, 2 * wordSize)));
11859 __ ldp(j_rarg1, j_rarg0, Address(__ post(sp, 2 * wordSize)));
11860
11861 __ ldpd(j_farg7, j_farg6, Address(__ post(sp, 2 * wordSize)));
11862 __ ldpd(j_farg5, j_farg4, Address(__ post(sp, 2 * wordSize)));
11863 __ ldpd(j_farg3, j_farg2, Address(__ post(sp, 2 * wordSize)));
11864 __ ldpd(j_farg1, j_farg0, Address(__ post(sp, 2 * wordSize)));
11865
11866 __ leave();
11867
11868 // check for pending exceptions
11869 Label pending;
11870 __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
11871 __ cbnz(rscratch1, pending);
11872
11873 if (has_res) {
11874 __ get_vm_result_oop(r0, rthread);
11875 }
11876
11877 __ ret(lr);
11878
11879 __ bind(pending);
11880 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
11881
11882 // -------------
11883 // make sure all code is generated
11884 masm->flush();
11885
11886 RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, frame_size_in_words, oop_maps, false);
11887 return stub->entry_point();
11888 }
11889
11890 // Initialization
11891 void generate_preuniverse_stubs() {
11892 // preuniverse stubs are not needed for aarch64
11893 }
11894
11895 void generate_initial_stubs() {
11896 // Generate initial stubs and initializes the entry points
11897
11898 // entry points that exist in all platforms Note: This is code
11899 // that could be shared among different platforms - however the
11900 // benefit seems to be smaller than the disadvantage of having a
11901 // much more complicated generator structure. See also comment in
11902 // stubRoutines.hpp.
11903
11904 StubRoutines::_forward_exception_entry = generate_forward_exception();
11905
11906 StubRoutines::_call_stub_entry =
11907 generate_call_stub(StubRoutines::_call_stub_return_address);
11908
11909 // is referenced by megamorphic call
11918 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
11919 }
11920
11921 if (UseCRC32CIntrinsics) {
11922 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
11923 }
11924
11925 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
11926 StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
11927 }
11928
11929 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
11930 StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
11931 }
11932
11933 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
11934 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
11935 StubRoutines::_hf2f = generate_float16ToFloat();
11936 StubRoutines::_f2hf = generate_floatToFloat16();
11937 }
11938
11939 if (InlineTypeReturnedAsFields) {
11940 StubRoutines::_load_inline_type_fields_in_regs =
11941 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::load_inline_type_fields_in_regs), "load_inline_type_fields_in_regs", false);
11942 StubRoutines::_store_inline_type_fields_to_buf =
11943 generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::store_inline_type_fields_to_buf), "store_inline_type_fields_to_buf", true);
11944 }
11945
11946 }
11947
11948 void generate_continuation_stubs() {
11949 // Continuation stubs:
11950 StubRoutines::_cont_thaw = generate_cont_thaw();
11951 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
11952 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
11953 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
11954 }
11955
11956 void generate_final_stubs() {
11957 // support for verify_oop (must happen after universe_init)
11958 if (VerifyOops) {
11959 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
11960 }
11961
11962 // arraycopy stubs used by compilers
11963 generate_arraycopy_stubs();
11964
11965 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
|