< prev index next >

src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp

Print this page

  311     __ mov(r19_sender_sp, sp);
  312     __ blr(c_rarg4);
  313 
  314     // we do this here because the notify will already have been done
  315     // if we get to the next instruction via an exception
  316     //
  317     // n.b. adding this instruction here affects the calculation of
  318     // whether or not a routine returns to the call stub (used when
  319     // doing stack walks) since the normal test is to check the return
  320     // pc against the address saved below. so we may need to allow for
  321     // this extra instruction in the check.
  322 
  323     // save current address for use by exception handling code
  324 
  325     return_address = __ pc();
  326 
  327     // store result depending on type (everything that is not
  328     // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
  329     // n.b. this assumes Java returns an integral result in r0
  330     // and a floating result in j_farg0
  331     __ ldr(j_rarg2, result);
  332     Label is_long, is_float, is_double, exit;
  333     __ ldr(j_rarg1, result_type);
  334     __ cmp(j_rarg1, (u1)T_OBJECT);







  335     __ br(Assembler::EQ, is_long);
  336     __ cmp(j_rarg1, (u1)T_LONG);
  337     __ br(Assembler::EQ, is_long);
  338     __ cmp(j_rarg1, (u1)T_FLOAT);
  339     __ br(Assembler::EQ, is_float);
  340     __ cmp(j_rarg1, (u1)T_DOUBLE);
  341     __ br(Assembler::EQ, is_double);
  342 
  343     // handle T_INT case
  344     __ strw(r0, Address(j_rarg2));
  345 
  346     __ BIND(exit);
  347 
  348     // pop parameters
  349     __ sub(esp, rfp, -sp_after_call_off * wordSize);
  350 
  351 #ifdef ASSERT
  352     // verify that threads correspond
  353     {
  354       Label L, S;
  355       __ ldr(rscratch1, thread);
  356       __ cmp(rthread, rscratch1);
  357       __ br(Assembler::NE, S);
  358       __ get_thread(rscratch1);
  359       __ cmp(rthread, rscratch1);
  360       __ br(Assembler::EQ, L);
  361       __ BIND(S);
  362       __ stop("StubRoutines::call_stub: threads must correspond");
  363       __ BIND(L);
  364     }

  376     __ ldp(r26, r25,   r26_save);
  377     __ ldp(r24, r23,   r24_save);
  378     __ ldp(r22, r21,   r22_save);
  379     __ ldp(r20, r19,   r20_save);
  380 
  381     // restore fpcr
  382     __ ldr(rscratch1,  fpcr_save);
  383     __ set_fpcr(rscratch1);
  384 
  385     __ ldp(c_rarg0, c_rarg1,  call_wrapper);
  386     __ ldrw(c_rarg2, result_type);
  387     __ ldr(c_rarg3,  method);
  388     __ ldp(c_rarg4, c_rarg5,  entry_point);
  389     __ ldp(c_rarg6, c_rarg7,  parameter_size);
  390 
  391     // leave frame and return to caller
  392     __ leave();
  393     __ ret(lr);
  394 
  395     // handle return types different from T_INT











  396 
  397     __ BIND(is_long);
  398     __ str(r0, Address(j_rarg2, 0));
  399     __ br(Assembler::AL, exit);
  400 
  401     __ BIND(is_float);
  402     __ strs(j_farg0, Address(j_rarg2, 0));
  403     __ br(Assembler::AL, exit);
  404 
  405     __ BIND(is_double);
  406     __ strd(j_farg0, Address(j_rarg2, 0));
  407     __ br(Assembler::AL, exit);
  408 
  409     return start;
  410   }
  411 
  412   // Return point for a Java call if there's an exception thrown in
  413   // Java code.  The exception is caught and transformed into a
  414   // pending exception stored in JavaThread that can be tested from
  415   // within the VM.
  416   //
  417   // Note: Usually the parameters are removed by the callee. In case
  418   // of an exception crossing an activation frame boundary, that is
  419   // not the case if the callee is compiled code => need to setup the
  420   // rsp.
  421   //
  422   // r0: exception oop
  423 
  424   address generate_catch_exception() {
  425     StubId stub_id = StubId::stubgen_catch_exception_id;
  426     StubCodeMark mark(this, stub_id);

 2208     //  |array_tag|     | header_size | element_type |     |log2_element_size|
 2209     // 32        30    24            16              8     2                 0
 2210     //
 2211     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
 2212     //
 2213 
 2214     const int lh_offset = in_bytes(Klass::layout_helper_offset());
 2215 
 2216     // Handle objArrays completely differently...
 2217     const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
 2218     __ ldrw(lh, Address(scratch_src_klass, lh_offset));
 2219     __ movw(rscratch1, objArray_lh);
 2220     __ eorw(rscratch2, lh, rscratch1);
 2221     __ cbzw(rscratch2, L_objArray);
 2222 
 2223     //  if (src->klass() != dst->klass()) return -1;
 2224     __ load_klass(rscratch2, dst);
 2225     __ eor(rscratch2, rscratch2, scratch_src_klass);
 2226     __ cbnz(rscratch2, L_failed);
 2227 






 2228     //  if (!src->is_Array()) return -1;
 2229     __ tbz(lh, 31, L_failed);  // i.e. (lh >= 0)
 2230 
 2231     // At this point, it is known to be a typeArray (array_tag 0x3).
 2232 #ifdef ASSERT
 2233     {
 2234       BLOCK_COMMENT("assert primitive array {");
 2235       Label L;
 2236       __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
 2237       __ cmpw(lh, rscratch2);
 2238       __ br(Assembler::GE, L);
 2239       __ stop("must be a primitive array");
 2240       __ bind(L);
 2241       BLOCK_COMMENT("} assert primitive array done");
 2242     }
 2243 #endif
 2244 
 2245     arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
 2246                            rscratch2, L_failed);
 2247 

10506     gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
10507 
10508     AtomicStubMark mark_cmpxchg_4_release
10509       (_masm, &aarch64_atomic_cmpxchg_4_release_impl);
10510     gen_cas_entry(MacroAssembler::word, memory_order_release);
10511     AtomicStubMark mark_cmpxchg_8_release
10512       (_masm, &aarch64_atomic_cmpxchg_8_release_impl);
10513     gen_cas_entry(MacroAssembler::xword, memory_order_release);
10514 
10515     AtomicStubMark mark_cmpxchg_4_seq_cst
10516       (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
10517     gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
10518     AtomicStubMark mark_cmpxchg_8_seq_cst
10519       (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
10520     gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
10521 
10522     ICache::invalidate_range(first_entry, __ pc() - first_entry);
10523   }
10524 #endif // LINUX
10525 
























10526   address generate_cont_thaw(Continuation::thaw_kind kind) {
10527     bool return_barrier = Continuation::is_thaw_return_barrier(kind);
10528     bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
10529 
10530     address start = __ pc();
10531 
10532     if (return_barrier) {
10533       __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
10534       __ mov(sp, rscratch1);
10535     }
10536     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
10537 
10538     if (return_barrier) {
10539       // preserve possible return value from a method returning to the return barrier
10540       __ fmovd(rscratch1, v0);
10541       __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
10542     }
10543 
10544     __ movw(c_rarg1, (return_barrier ? 1 : 0));
10545     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
10546     __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
10547 
10548     if (return_barrier) {
10549       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
10550       __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
10551       __ fmovd(v0, rscratch1);
10552     }
10553     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
10554 
10555 
10556     Label thaw_success;
10557     // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
10558     __ cbnz(rscratch2, thaw_success);
10559     __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
10560     __ br(rscratch1);
10561     __ bind(thaw_success);
10562 
10563     // make room for the thawed frames
10564     __ sub(rscratch1, sp, rscratch2);
10565     __ andr(rscratch1, rscratch1, -16); // align
10566     __ mov(sp, rscratch1);
10567 
10568     if (return_barrier) {
10569       // save original return value -- again
10570       __ fmovd(rscratch1, v0);
10571       __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
10572     }
10573 
10574     // If we want, we can templatize thaw by kind, and have three different entries
10575     __ movw(c_rarg1, (uint32_t)kind);
10576 
10577     __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
10578     __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
10579 
10580     if (return_barrier) {
10581       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
10582       __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
10583       __ fmovd(v0, rscratch1);
10584     } else {
10585       __ mov(r0, zr); // return 0 (success) from doYield
10586     }
10587 
10588     // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
10589     __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
10590     __ mov(rfp, sp);
10591 
10592     if (return_barrier_exception) {
10593       __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
10594       __ authenticate_return_address(c_rarg1);
10595       __ verify_oop(r0);
10596       // save return value containing the exception oop in callee-saved R19
10597       __ mov(r19, r0);
10598 
10599       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
10600 
10601       // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
10602       // __ reinitialize_ptrue();
10603 

11700     //       assert(Ra == Pa_base[j], "must be");
11701     //       MACC(Ra, Ra, t0, t1, t2);
11702     //     }
11703     //     iters =  (2*len-i)/2;
11704     //     assert(iters == len-j, "must be");
11705     //     for (; iters--; j++) {
11706     //       assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
11707     //       MACC(Rm, Rn, t0, t1, t2);
11708     //       Rm = *++Pm;
11709     //       Rn = *--Pn;
11710     //     }
11711     //     Pm_base[i-len] = t0;
11712     //     t0 = t1; t1 = t2; t2 = 0;
11713     //   }
11714 
11715     //   while (t0)
11716     //     t0 = sub(Pm_base, Pn_base, t0, len);
11717     // }
11718   };
11719 
































































































































11720   // Initialization
11721   void generate_preuniverse_stubs() {
11722     // preuniverse stubs are not needed for aarch64
11723   }
11724 
11725   void generate_initial_stubs() {
11726     // Generate initial stubs and initializes the entry points
11727 
11728     // entry points that exist in all platforms Note: This is code
11729     // that could be shared among different platforms - however the
11730     // benefit seems to be smaller than the disadvantage of having a
11731     // much more complicated generator structure. See also comment in
11732     // stubRoutines.hpp.
11733 
11734     StubRoutines::_forward_exception_entry = generate_forward_exception();
11735 
11736     StubRoutines::_call_stub_entry =
11737       generate_call_stub(StubRoutines::_call_stub_return_address);
11738 
11739     // is referenced by megamorphic call

11748       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
11749     }
11750 
11751     if (UseCRC32CIntrinsics) {
11752       StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
11753     }
11754 
11755     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
11756       StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
11757     }
11758 
11759     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
11760       StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
11761     }
11762 
11763     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
11764         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
11765       StubRoutines::_hf2f = generate_float16ToFloat();
11766       StubRoutines::_f2hf = generate_floatToFloat16();
11767     }








11768   }
11769 
11770   void generate_continuation_stubs() {
11771     // Continuation stubs:
11772     StubRoutines::_cont_thaw          = generate_cont_thaw();
11773     StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
11774     StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
11775     StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
11776   }
11777 
11778   void generate_final_stubs() {
11779     // support for verify_oop (must happen after universe_init)
11780     if (VerifyOops) {
11781       StubRoutines::_verify_oop_subroutine_entry   = generate_verify_oop();
11782     }
11783 
11784     // arraycopy stubs used by compilers
11785     generate_arraycopy_stubs();
11786 
11787     StubRoutines::_method_entry_barrier = generate_method_entry_barrier();

  311     __ mov(r19_sender_sp, sp);
  312     __ blr(c_rarg4);
  313 
  314     // we do this here because the notify will already have been done
  315     // if we get to the next instruction via an exception
  316     //
  317     // n.b. adding this instruction here affects the calculation of
  318     // whether or not a routine returns to the call stub (used when
  319     // doing stack walks) since the normal test is to check the return
  320     // pc against the address saved below. so we may need to allow for
  321     // this extra instruction in the check.
  322 
  323     // save current address for use by exception handling code
  324 
  325     return_address = __ pc();
  326 
  327     // store result depending on type (everything that is not
  328     // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
  329     // n.b. this assumes Java returns an integral result in r0
  330     // and a floating result in j_farg0
  331     // All of j_rargN may be used to return inline type fields so be careful
  332     // not to clobber those.
  333     // SharedRuntime::generate_buffered_inline_type_adapter() knows the register
  334     // assignment of Rresult below.
  335     Register Rresult = r14, Rresult_type = r15;
  336     __ ldr(Rresult, result);
  337     Label is_long, is_float, is_double, check_prim, exit;
  338     __ ldr(Rresult_type, result_type);
  339     __ cmp(Rresult_type, (u1)T_OBJECT);
  340     __ br(Assembler::EQ, check_prim);
  341     __ cmp(Rresult_type, (u1)T_LONG);
  342     __ br(Assembler::EQ, is_long);
  343     __ cmp(Rresult_type, (u1)T_FLOAT);


  344     __ br(Assembler::EQ, is_float);
  345     __ cmp(Rresult_type, (u1)T_DOUBLE);
  346     __ br(Assembler::EQ, is_double);
  347 
  348     // handle T_INT case
  349     __ strw(r0, Address(Rresult));
  350 
  351     __ BIND(exit);
  352 
  353     // pop parameters
  354     __ sub(esp, rfp, -sp_after_call_off * wordSize);
  355 
  356 #ifdef ASSERT
  357     // verify that threads correspond
  358     {
  359       Label L, S;
  360       __ ldr(rscratch1, thread);
  361       __ cmp(rthread, rscratch1);
  362       __ br(Assembler::NE, S);
  363       __ get_thread(rscratch1);
  364       __ cmp(rthread, rscratch1);
  365       __ br(Assembler::EQ, L);
  366       __ BIND(S);
  367       __ stop("StubRoutines::call_stub: threads must correspond");
  368       __ BIND(L);
  369     }

  381     __ ldp(r26, r25,   r26_save);
  382     __ ldp(r24, r23,   r24_save);
  383     __ ldp(r22, r21,   r22_save);
  384     __ ldp(r20, r19,   r20_save);
  385 
  386     // restore fpcr
  387     __ ldr(rscratch1,  fpcr_save);
  388     __ set_fpcr(rscratch1);
  389 
  390     __ ldp(c_rarg0, c_rarg1,  call_wrapper);
  391     __ ldrw(c_rarg2, result_type);
  392     __ ldr(c_rarg3,  method);
  393     __ ldp(c_rarg4, c_rarg5,  entry_point);
  394     __ ldp(c_rarg6, c_rarg7,  parameter_size);
  395 
  396     // leave frame and return to caller
  397     __ leave();
  398     __ ret(lr);
  399 
  400     // handle return types different from T_INT
  401     __ BIND(check_prim);
  402     if (InlineTypeReturnedAsFields) {
  403       // Check for scalarized return value
  404       __ tbz(r0, 0, is_long);
  405       // Load pack handler address
  406       __ andr(rscratch1, r0, -2);
  407       __ ldr(rscratch1, Address(rscratch1, InstanceKlass::adr_inlineklass_fixed_block_offset()));
  408       __ ldr(rscratch1, Address(rscratch1, InlineKlass::pack_handler_jobject_offset()));
  409       __ blr(rscratch1);
  410       __ b(exit);
  411     }
  412 
  413     __ BIND(is_long);
  414     __ str(r0, Address(Rresult, 0));
  415     __ br(Assembler::AL, exit);
  416 
  417     __ BIND(is_float);
  418     __ strs(j_farg0, Address(Rresult, 0));
  419     __ br(Assembler::AL, exit);
  420 
  421     __ BIND(is_double);
  422     __ strd(j_farg0, Address(Rresult, 0));
  423     __ br(Assembler::AL, exit);
  424 
  425     return start;
  426   }
  427 
  428   // Return point for a Java call if there's an exception thrown in
  429   // Java code.  The exception is caught and transformed into a
  430   // pending exception stored in JavaThread that can be tested from
  431   // within the VM.
  432   //
  433   // Note: Usually the parameters are removed by the callee. In case
  434   // of an exception crossing an activation frame boundary, that is
  435   // not the case if the callee is compiled code => need to setup the
  436   // rsp.
  437   //
  438   // r0: exception oop
  439 
  440   address generate_catch_exception() {
  441     StubId stub_id = StubId::stubgen_catch_exception_id;
  442     StubCodeMark mark(this, stub_id);

 2224     //  |array_tag|     | header_size | element_type |     |log2_element_size|
 2225     // 32        30    24            16              8     2                 0
 2226     //
 2227     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
 2228     //
 2229 
 2230     const int lh_offset = in_bytes(Klass::layout_helper_offset());
 2231 
 2232     // Handle objArrays completely differently...
 2233     const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
 2234     __ ldrw(lh, Address(scratch_src_klass, lh_offset));
 2235     __ movw(rscratch1, objArray_lh);
 2236     __ eorw(rscratch2, lh, rscratch1);
 2237     __ cbzw(rscratch2, L_objArray);
 2238 
 2239     //  if (src->klass() != dst->klass()) return -1;
 2240     __ load_klass(rscratch2, dst);
 2241     __ eor(rscratch2, rscratch2, scratch_src_klass);
 2242     __ cbnz(rscratch2, L_failed);
 2243 
 2244     // Check for flat inline type array -> return -1
 2245     __ test_flat_array_oop(src, rscratch2, L_failed);
 2246 
 2247     // Check for null-free (non-flat) inline type array -> handle as object array
 2248     __ test_null_free_array_oop(src, rscratch2, L_objArray);
 2249 
 2250     //  if (!src->is_Array()) return -1;
 2251     __ tbz(lh, 31, L_failed);  // i.e. (lh >= 0)
 2252 
 2253     // At this point, it is known to be a typeArray (array_tag 0x3).
 2254 #ifdef ASSERT
 2255     {
 2256       BLOCK_COMMENT("assert primitive array {");
 2257       Label L;
 2258       __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
 2259       __ cmpw(lh, rscratch2);
 2260       __ br(Assembler::GE, L);
 2261       __ stop("must be a primitive array");
 2262       __ bind(L);
 2263       BLOCK_COMMENT("} assert primitive array done");
 2264     }
 2265 #endif
 2266 
 2267     arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
 2268                            rscratch2, L_failed);
 2269 

10528     gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
10529 
10530     AtomicStubMark mark_cmpxchg_4_release
10531       (_masm, &aarch64_atomic_cmpxchg_4_release_impl);
10532     gen_cas_entry(MacroAssembler::word, memory_order_release);
10533     AtomicStubMark mark_cmpxchg_8_release
10534       (_masm, &aarch64_atomic_cmpxchg_8_release_impl);
10535     gen_cas_entry(MacroAssembler::xword, memory_order_release);
10536 
10537     AtomicStubMark mark_cmpxchg_4_seq_cst
10538       (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
10539     gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
10540     AtomicStubMark mark_cmpxchg_8_seq_cst
10541       (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
10542     gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
10543 
10544     ICache::invalidate_range(first_entry, __ pc() - first_entry);
10545   }
10546 #endif // LINUX
10547 
10548   static void save_return_registers(MacroAssembler* masm) {
10549     if (InlineTypeReturnedAsFields) {
10550       masm->push(RegSet::range(r0, r7), sp);
10551       masm->sub(sp, sp, 4 * wordSize);
10552       masm->st1(v0, v1, v2, v3, masm->T1D, Address(sp));
10553       masm->sub(sp, sp, 4 * wordSize);
10554       masm->st1(v4, v5, v6, v7, masm->T1D, Address(sp));
10555     } else {
10556       masm->fmovd(rscratch1, v0);
10557       masm->stp(rscratch1, r0, Address(masm->pre(sp, -2 * wordSize)));
10558     }
10559   }
10560 
10561   static void restore_return_registers(MacroAssembler* masm) {
10562     if (InlineTypeReturnedAsFields) {
10563       masm->ld1(v4, v5, v6, v7, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
10564       masm->ld1(v0, v1, v2, v3, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
10565       masm->pop(RegSet::range(r0, r7), sp);
10566     } else {
10567       masm->ldp(rscratch1, r0, Address(masm->post(sp, 2 * wordSize)));
10568       masm->fmovd(v0, rscratch1);
10569     }
10570   }
10571 
10572   address generate_cont_thaw(Continuation::thaw_kind kind) {
10573     bool return_barrier = Continuation::is_thaw_return_barrier(kind);
10574     bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
10575 
10576     address start = __ pc();
10577 
10578     if (return_barrier) {
10579       __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
10580       __ mov(sp, rscratch1);
10581     }
10582     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
10583 
10584     if (return_barrier) {
10585       // preserve possible return value from a method returning to the return barrier
10586       save_return_registers(_masm);

10587     }
10588 
10589     __ movw(c_rarg1, (return_barrier ? 1 : 0));
10590     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
10591     __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
10592 
10593     if (return_barrier) {
10594       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
10595       restore_return_registers(_masm);

10596     }
10597     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
10598 
10599 
10600     Label thaw_success;
10601     // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
10602     __ cbnz(rscratch2, thaw_success);
10603     __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
10604     __ br(rscratch1);
10605     __ bind(thaw_success);
10606 
10607     // make room for the thawed frames
10608     __ sub(rscratch1, sp, rscratch2);
10609     __ andr(rscratch1, rscratch1, -16); // align
10610     __ mov(sp, rscratch1);
10611 
10612     if (return_barrier) {
10613       // save original return value -- again
10614       save_return_registers(_masm);

10615     }
10616 
10617     // If we want, we can templatize thaw by kind, and have three different entries
10618     __ movw(c_rarg1, (uint32_t)kind);
10619 
10620     __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
10621     __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
10622 
10623     if (return_barrier) {
10624       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
10625       restore_return_registers(_masm);

10626     } else {
10627       __ mov(r0, zr); // return 0 (success) from doYield
10628     }
10629 
10630     // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
10631     __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
10632     __ mov(rfp, sp);
10633 
10634     if (return_barrier_exception) {
10635       __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
10636       __ authenticate_return_address(c_rarg1);
10637       __ verify_oop(r0);
10638       // save return value containing the exception oop in callee-saved R19
10639       __ mov(r19, r0);
10640 
10641       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
10642 
10643       // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
10644       // __ reinitialize_ptrue();
10645 

11742     //       assert(Ra == Pa_base[j], "must be");
11743     //       MACC(Ra, Ra, t0, t1, t2);
11744     //     }
11745     //     iters =  (2*len-i)/2;
11746     //     assert(iters == len-j, "must be");
11747     //     for (; iters--; j++) {
11748     //       assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
11749     //       MACC(Rm, Rn, t0, t1, t2);
11750     //       Rm = *++Pm;
11751     //       Rn = *--Pn;
11752     //     }
11753     //     Pm_base[i-len] = t0;
11754     //     t0 = t1; t1 = t2; t2 = 0;
11755     //   }
11756 
11757     //   while (t0)
11758     //     t0 = sub(Pm_base, Pn_base, t0, len);
11759     // }
11760   };
11761 
11762   // Call here from the interpreter or compiled code to either load
11763   // multiple returned values from the inline type instance being
11764   // returned to registers or to store returned values to a newly
11765   // allocated inline type instance.
11766   address generate_return_value_stub(address destination, const char* name, bool has_res) {
11767     // We need to save all registers the calling convention may use so
11768     // the runtime calls read or update those registers. This needs to
11769     // be in sync with SharedRuntime::java_return_convention().
11770     // n.b. aarch64 asserts that frame::arg_reg_save_area_bytes == 0
11771     enum layout {
11772       j_rarg7_off = 0, j_rarg7_2,    // j_rarg7 is r0
11773       j_rarg6_off, j_rarg6_2,
11774       j_rarg5_off, j_rarg5_2,
11775       j_rarg4_off, j_rarg4_2,
11776       j_rarg3_off, j_rarg3_2,
11777       j_rarg2_off, j_rarg2_2,
11778       j_rarg1_off, j_rarg1_2,
11779       j_rarg0_off, j_rarg0_2,
11780 
11781       j_farg7_off, j_farg7_2,
11782       j_farg6_off, j_farg6_2,
11783       j_farg5_off, j_farg5_2,
11784       j_farg4_off, j_farg4_2,
11785       j_farg3_off, j_farg3_2,
11786       j_farg2_off, j_farg2_2,
11787       j_farg1_off, j_farg1_2,
11788       j_farg0_off, j_farg0_2,
11789 
11790       rfp_off, rfp_off2,
11791       return_off, return_off2,
11792 
11793       framesize // inclusive of return address
11794     };
11795 
11796     CodeBuffer code(name, 512, 64);
11797     MacroAssembler* masm = new MacroAssembler(&code);
11798 
11799     int frame_size_in_bytes = align_up(framesize*BytesPerInt, 16);
11800     assert(frame_size_in_bytes == framesize*BytesPerInt, "misaligned");
11801     int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
11802     int frame_size_in_words = frame_size_in_bytes / wordSize;
11803 
11804     OopMapSet* oop_maps = new OopMapSet();
11805     OopMap* map = new OopMap(frame_size_in_slots, 0);
11806 
11807     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg7_off), j_rarg7->as_VMReg());
11808     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg6_off), j_rarg6->as_VMReg());
11809     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg5_off), j_rarg5->as_VMReg());
11810     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg4_off), j_rarg4->as_VMReg());
11811     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg3_off), j_rarg3->as_VMReg());
11812     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg2_off), j_rarg2->as_VMReg());
11813     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg1_off), j_rarg1->as_VMReg());
11814     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg0_off), j_rarg0->as_VMReg());
11815 
11816     map->set_callee_saved(VMRegImpl::stack2reg(j_farg0_off), j_farg0->as_VMReg());
11817     map->set_callee_saved(VMRegImpl::stack2reg(j_farg1_off), j_farg1->as_VMReg());
11818     map->set_callee_saved(VMRegImpl::stack2reg(j_farg2_off), j_farg2->as_VMReg());
11819     map->set_callee_saved(VMRegImpl::stack2reg(j_farg3_off), j_farg3->as_VMReg());
11820     map->set_callee_saved(VMRegImpl::stack2reg(j_farg4_off), j_farg4->as_VMReg());
11821     map->set_callee_saved(VMRegImpl::stack2reg(j_farg5_off), j_farg5->as_VMReg());
11822     map->set_callee_saved(VMRegImpl::stack2reg(j_farg6_off), j_farg6->as_VMReg());
11823     map->set_callee_saved(VMRegImpl::stack2reg(j_farg7_off), j_farg7->as_VMReg());
11824 
11825     address start = __ pc();
11826 
11827     __ enter(); // Save FP and LR before call
11828 
11829     __ stpd(j_farg1, j_farg0, Address(__ pre(sp, -2 * wordSize)));
11830     __ stpd(j_farg3, j_farg2, Address(__ pre(sp, -2 * wordSize)));
11831     __ stpd(j_farg5, j_farg4, Address(__ pre(sp, -2 * wordSize)));
11832     __ stpd(j_farg7, j_farg6, Address(__ pre(sp, -2 * wordSize)));
11833 
11834     __ stp(j_rarg1, j_rarg0, Address(__ pre(sp, -2 * wordSize)));
11835     __ stp(j_rarg3, j_rarg2, Address(__ pre(sp, -2 * wordSize)));
11836     __ stp(j_rarg5, j_rarg4, Address(__ pre(sp, -2 * wordSize)));
11837     __ stp(j_rarg7, j_rarg6, Address(__ pre(sp, -2 * wordSize)));
11838 
11839     int frame_complete = __ offset();
11840 
11841     // Set up last_Java_sp and last_Java_fp
11842     address the_pc = __ pc();
11843     __ set_last_Java_frame(sp, noreg, the_pc, rscratch1);
11844 
11845     // Call runtime
11846     __ mov(c_rarg1, r0);
11847     __ mov(c_rarg0, rthread);
11848 
11849     __ mov(rscratch1, destination);
11850     __ blr(rscratch1);
11851 
11852     oop_maps->add_gc_map(the_pc - start, map);
11853 
11854     __ reset_last_Java_frame(false);
11855 
11856     __ ldp(j_rarg7, j_rarg6, Address(__ post(sp, 2 * wordSize)));
11857     __ ldp(j_rarg5, j_rarg4, Address(__ post(sp, 2 * wordSize)));
11858     __ ldp(j_rarg3, j_rarg2, Address(__ post(sp, 2 * wordSize)));
11859     __ ldp(j_rarg1, j_rarg0, Address(__ post(sp, 2 * wordSize)));
11860 
11861     __ ldpd(j_farg7, j_farg6, Address(__ post(sp, 2 * wordSize)));
11862     __ ldpd(j_farg5, j_farg4, Address(__ post(sp, 2 * wordSize)));
11863     __ ldpd(j_farg3, j_farg2, Address(__ post(sp, 2 * wordSize)));
11864     __ ldpd(j_farg1, j_farg0, Address(__ post(sp, 2 * wordSize)));
11865 
11866     __ leave();
11867 
11868     // check for pending exceptions
11869     Label pending;
11870     __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
11871     __ cbnz(rscratch1, pending);
11872 
11873     if (has_res) {
11874       __ get_vm_result_oop(r0, rthread);
11875     }
11876 
11877     __ ret(lr);
11878 
11879     __ bind(pending);
11880     __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
11881 
11882     // -------------
11883     // make sure all code is generated
11884     masm->flush();
11885 
11886     RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, frame_size_in_words, oop_maps, false);
11887     return stub->entry_point();
11888   }
11889 
11890   // Initialization
11891   void generate_preuniverse_stubs() {
11892     // preuniverse stubs are not needed for aarch64
11893   }
11894 
11895   void generate_initial_stubs() {
11896     // Generate initial stubs and initializes the entry points
11897 
11898     // entry points that exist in all platforms Note: This is code
11899     // that could be shared among different platforms - however the
11900     // benefit seems to be smaller than the disadvantage of having a
11901     // much more complicated generator structure. See also comment in
11902     // stubRoutines.hpp.
11903 
11904     StubRoutines::_forward_exception_entry = generate_forward_exception();
11905 
11906     StubRoutines::_call_stub_entry =
11907       generate_call_stub(StubRoutines::_call_stub_return_address);
11908 
11909     // is referenced by megamorphic call

11918       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
11919     }
11920 
11921     if (UseCRC32CIntrinsics) {
11922       StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
11923     }
11924 
11925     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
11926       StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
11927     }
11928 
11929     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
11930       StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
11931     }
11932 
11933     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
11934         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
11935       StubRoutines::_hf2f = generate_float16ToFloat();
11936       StubRoutines::_f2hf = generate_floatToFloat16();
11937     }
11938 
11939     if (InlineTypeReturnedAsFields) {
11940       StubRoutines::_load_inline_type_fields_in_regs =
11941          generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::load_inline_type_fields_in_regs), "load_inline_type_fields_in_regs", false);
11942       StubRoutines::_store_inline_type_fields_to_buf =
11943          generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::store_inline_type_fields_to_buf), "store_inline_type_fields_to_buf", true);
11944     }
11945 
11946   }
11947 
11948   void generate_continuation_stubs() {
11949     // Continuation stubs:
11950     StubRoutines::_cont_thaw          = generate_cont_thaw();
11951     StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
11952     StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
11953     StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
11954   }
11955 
11956   void generate_final_stubs() {
11957     // support for verify_oop (must happen after universe_init)
11958     if (VerifyOops) {
11959       StubRoutines::_verify_oop_subroutine_entry   = generate_verify_oop();
11960     }
11961 
11962     // arraycopy stubs used by compilers
11963     generate_arraycopy_stubs();
11964 
11965     StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
< prev index next >