< prev index next >

src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp

Print this page

  311     __ mov(r19_sender_sp, sp);
  312     __ blr(c_rarg4);
  313 
  314     // we do this here because the notify will already have been done
  315     // if we get to the next instruction via an exception
  316     //
  317     // n.b. adding this instruction here affects the calculation of
  318     // whether or not a routine returns to the call stub (used when
  319     // doing stack walks) since the normal test is to check the return
  320     // pc against the address saved below. so we may need to allow for
  321     // this extra instruction in the check.
  322 
  323     // save current address for use by exception handling code
  324 
  325     return_address = __ pc();
  326 
  327     // store result depending on type (everything that is not
  328     // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
  329     // n.b. this assumes Java returns an integral result in r0
  330     // and a floating result in j_farg0
  331     __ ldr(j_rarg2, result);
  332     Label is_long, is_float, is_double, exit;
  333     __ ldr(j_rarg1, result_type);
  334     __ cmp(j_rarg1, (u1)T_OBJECT);







  335     __ br(Assembler::EQ, is_long);
  336     __ cmp(j_rarg1, (u1)T_LONG);
  337     __ br(Assembler::EQ, is_long);
  338     __ cmp(j_rarg1, (u1)T_FLOAT);
  339     __ br(Assembler::EQ, is_float);
  340     __ cmp(j_rarg1, (u1)T_DOUBLE);
  341     __ br(Assembler::EQ, is_double);
  342 
  343     // handle T_INT case
  344     __ strw(r0, Address(j_rarg2));
  345 
  346     __ BIND(exit);
  347 
  348     // pop parameters
  349     __ sub(esp, rfp, -sp_after_call_off * wordSize);
  350 
  351 #ifdef ASSERT
  352     // verify that threads correspond
  353     {
  354       Label L, S;
  355       __ ldr(rscratch1, thread);
  356       __ cmp(rthread, rscratch1);
  357       __ br(Assembler::NE, S);
  358       __ get_thread(rscratch1);
  359       __ cmp(rthread, rscratch1);
  360       __ br(Assembler::EQ, L);
  361       __ BIND(S);
  362       __ stop("StubRoutines::call_stub: threads must correspond");
  363       __ BIND(L);
  364     }

  376     __ ldp(r26, r25,   r26_save);
  377     __ ldp(r24, r23,   r24_save);
  378     __ ldp(r22, r21,   r22_save);
  379     __ ldp(r20, r19,   r20_save);
  380 
  381     // restore fpcr
  382     __ ldr(rscratch1,  fpcr_save);
  383     __ set_fpcr(rscratch1);
  384 
  385     __ ldp(c_rarg0, c_rarg1,  call_wrapper);
  386     __ ldrw(c_rarg2, result_type);
  387     __ ldr(c_rarg3,  method);
  388     __ ldp(c_rarg4, c_rarg5,  entry_point);
  389     __ ldp(c_rarg6, c_rarg7,  parameter_size);
  390 
  391     // leave frame and return to caller
  392     __ leave();
  393     __ ret(lr);
  394 
  395     // handle return types different from T_INT











  396 
  397     __ BIND(is_long);
  398     __ str(r0, Address(j_rarg2, 0));
  399     __ br(Assembler::AL, exit);
  400 
  401     __ BIND(is_float);
  402     __ strs(j_farg0, Address(j_rarg2, 0));
  403     __ br(Assembler::AL, exit);
  404 
  405     __ BIND(is_double);
  406     __ strd(j_farg0, Address(j_rarg2, 0));
  407     __ br(Assembler::AL, exit);
  408 
  409     return start;
  410   }
  411 
  412   // Return point for a Java call if there's an exception thrown in
  413   // Java code.  The exception is caught and transformed into a
  414   // pending exception stored in JavaThread that can be tested from
  415   // within the VM.
  416   //
  417   // Note: Usually the parameters are removed by the callee. In case
  418   // of an exception crossing an activation frame boundary, that is
  419   // not the case if the callee is compiled code => need to setup the
  420   // rsp.
  421   //
  422   // r0: exception oop
  423 
  424   address generate_catch_exception() {
  425     StubId stub_id = StubId::stubgen_catch_exception_id;
  426     StubCodeMark mark(this, stub_id);

 2207     //  |array_tag|     | header_size | element_type |     |log2_element_size|
 2208     // 32        30    24            16              8     2                 0
 2209     //
 2210     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
 2211     //
 2212 
 2213     const int lh_offset = in_bytes(Klass::layout_helper_offset());
 2214 
 2215     // Handle objArrays completely differently...
 2216     const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
 2217     __ ldrw(lh, Address(scratch_src_klass, lh_offset));
 2218     __ movw(rscratch1, objArray_lh);
 2219     __ eorw(rscratch2, lh, rscratch1);
 2220     __ cbzw(rscratch2, L_objArray);
 2221 
 2222     //  if (src->klass() != dst->klass()) return -1;
 2223     __ load_klass(rscratch2, dst);
 2224     __ eor(rscratch2, rscratch2, scratch_src_klass);
 2225     __ cbnz(rscratch2, L_failed);
 2226 






 2227     //  if (!src->is_Array()) return -1;
 2228     __ tbz(lh, 31, L_failed);  // i.e. (lh >= 0)
 2229 
 2230     // At this point, it is known to be a typeArray (array_tag 0x3).
 2231 #ifdef ASSERT
 2232     {
 2233       BLOCK_COMMENT("assert primitive array {");
 2234       Label L;
 2235       __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
 2236       __ cmpw(lh, rscratch2);
 2237       __ br(Assembler::GE, L);
 2238       __ stop("must be a primitive array");
 2239       __ bind(L);
 2240       BLOCK_COMMENT("} assert primitive array done");
 2241     }
 2242 #endif
 2243 
 2244     arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
 2245                            rscratch2, L_failed);
 2246 

10444     gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
10445 
10446     AtomicStubMark mark_cmpxchg_4_release
10447       (_masm, &aarch64_atomic_cmpxchg_4_release_impl);
10448     gen_cas_entry(MacroAssembler::word, memory_order_release);
10449     AtomicStubMark mark_cmpxchg_8_release
10450       (_masm, &aarch64_atomic_cmpxchg_8_release_impl);
10451     gen_cas_entry(MacroAssembler::xword, memory_order_release);
10452 
10453     AtomicStubMark mark_cmpxchg_4_seq_cst
10454       (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
10455     gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
10456     AtomicStubMark mark_cmpxchg_8_seq_cst
10457       (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
10458     gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
10459 
10460     ICache::invalidate_range(first_entry, __ pc() - first_entry);
10461   }
10462 #endif // LINUX
10463 
























10464   address generate_cont_thaw(Continuation::thaw_kind kind) {
10465     bool return_barrier = Continuation::is_thaw_return_barrier(kind);
10466     bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
10467 
10468     address start = __ pc();
10469 
10470     if (return_barrier) {
10471       __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
10472       __ mov(sp, rscratch1);
10473     }
10474     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
10475 
10476     if (return_barrier) {
10477       // preserve possible return value from a method returning to the return barrier
10478       __ fmovd(rscratch1, v0);
10479       __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
10480     }
10481 
10482     __ movw(c_rarg1, (return_barrier ? 1 : 0));
10483     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
10484     __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
10485 
10486     if (return_barrier) {
10487       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
10488       __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
10489       __ fmovd(v0, rscratch1);
10490     }
10491     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
10492 
10493 
10494     Label thaw_success;
10495     // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
10496     __ cbnz(rscratch2, thaw_success);
10497     __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
10498     __ br(rscratch1);
10499     __ bind(thaw_success);
10500 
10501     // make room for the thawed frames
10502     __ sub(rscratch1, sp, rscratch2);
10503     __ andr(rscratch1, rscratch1, -16); // align
10504     __ mov(sp, rscratch1);
10505 
10506     if (return_barrier) {
10507       // save original return value -- again
10508       __ fmovd(rscratch1, v0);
10509       __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
10510     }
10511 
10512     // If we want, we can templatize thaw by kind, and have three different entries
10513     __ movw(c_rarg1, (uint32_t)kind);
10514 
10515     __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
10516     __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
10517 
10518     if (return_barrier) {
10519       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
10520       __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
10521       __ fmovd(v0, rscratch1);
10522     } else {
10523       __ mov(r0, zr); // return 0 (success) from doYield
10524     }
10525 
10526     // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
10527     __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
10528     __ mov(rfp, sp);
10529 
10530     if (return_barrier_exception) {
10531       __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
10532       __ authenticate_return_address(c_rarg1);
10533       __ verify_oop(r0);
10534       // save return value containing the exception oop in callee-saved R19
10535       __ mov(r19, r0);
10536 
10537       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
10538 
10539       // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
10540       // __ reinitialize_ptrue();
10541 

11638     //       assert(Ra == Pa_base[j], "must be");
11639     //       MACC(Ra, Ra, t0, t1, t2);
11640     //     }
11641     //     iters =  (2*len-i)/2;
11642     //     assert(iters == len-j, "must be");
11643     //     for (; iters--; j++) {
11644     //       assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
11645     //       MACC(Rm, Rn, t0, t1, t2);
11646     //       Rm = *++Pm;
11647     //       Rn = *--Pn;
11648     //     }
11649     //     Pm_base[i-len] = t0;
11650     //     t0 = t1; t1 = t2; t2 = 0;
11651     //   }
11652 
11653     //   while (t0)
11654     //     t0 = sub(Pm_base, Pn_base, t0, len);
11655     // }
11656   };
11657 
































































































































11658   // Initialization
11659   void generate_preuniverse_stubs() {
11660     // preuniverse stubs are not needed for aarch64
11661   }
11662 
11663   void generate_initial_stubs() {
11664     // Generate initial stubs and initializes the entry points
11665 
11666     // entry points that exist in all platforms Note: This is code
11667     // that could be shared among different platforms - however the
11668     // benefit seems to be smaller than the disadvantage of having a
11669     // much more complicated generator structure. See also comment in
11670     // stubRoutines.hpp.
11671 
11672     StubRoutines::_forward_exception_entry = generate_forward_exception();
11673 
11674     StubRoutines::_call_stub_entry =
11675       generate_call_stub(StubRoutines::_call_stub_return_address);
11676 
11677     // is referenced by megamorphic call

11686       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
11687     }
11688 
11689     if (UseCRC32CIntrinsics) {
11690       StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
11691     }
11692 
11693     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
11694       StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
11695     }
11696 
11697     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
11698       StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
11699     }
11700 
11701     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
11702         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
11703       StubRoutines::_hf2f = generate_float16ToFloat();
11704       StubRoutines::_f2hf = generate_floatToFloat16();
11705     }








11706   }
11707 
11708   void generate_continuation_stubs() {
11709     // Continuation stubs:
11710     StubRoutines::_cont_thaw          = generate_cont_thaw();
11711     StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
11712     StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
11713     StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
11714   }
11715 
11716   void generate_final_stubs() {
11717     // support for verify_oop (must happen after universe_init)
11718     if (VerifyOops) {
11719       StubRoutines::_verify_oop_subroutine_entry   = generate_verify_oop();
11720     }
11721 
11722     // arraycopy stubs used by compilers
11723     generate_arraycopy_stubs();
11724 
11725     StubRoutines::_method_entry_barrier = generate_method_entry_barrier();

  311     __ mov(r19_sender_sp, sp);
  312     __ blr(c_rarg4);
  313 
  314     // we do this here because the notify will already have been done
  315     // if we get to the next instruction via an exception
  316     //
  317     // n.b. adding this instruction here affects the calculation of
  318     // whether or not a routine returns to the call stub (used when
  319     // doing stack walks) since the normal test is to check the return
  320     // pc against the address saved below. so we may need to allow for
  321     // this extra instruction in the check.
  322 
  323     // save current address for use by exception handling code
  324 
  325     return_address = __ pc();
  326 
  327     // store result depending on type (everything that is not
  328     // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
  329     // n.b. this assumes Java returns an integral result in r0
  330     // and a floating result in j_farg0
  331     // All of j_rargN may be used to return inline type fields so be careful
  332     // not to clobber those.
  333     // SharedRuntime::generate_buffered_inline_type_adapter() knows the register
  334     // assignment of Rresult below.
  335     Register Rresult = r14, Rresult_type = r15;
  336     __ ldr(Rresult, result);
  337     Label is_long, is_float, is_double, check_prim, exit;
  338     __ ldr(Rresult_type, result_type);
  339     __ cmp(Rresult_type, (u1)T_OBJECT);
  340     __ br(Assembler::EQ, check_prim);
  341     __ cmp(Rresult_type, (u1)T_LONG);
  342     __ br(Assembler::EQ, is_long);
  343     __ cmp(Rresult_type, (u1)T_FLOAT);


  344     __ br(Assembler::EQ, is_float);
  345     __ cmp(Rresult_type, (u1)T_DOUBLE);
  346     __ br(Assembler::EQ, is_double);
  347 
  348     // handle T_INT case
  349     __ strw(r0, Address(Rresult));
  350 
  351     __ BIND(exit);
  352 
  353     // pop parameters
  354     __ sub(esp, rfp, -sp_after_call_off * wordSize);
  355 
  356 #ifdef ASSERT
  357     // verify that threads correspond
  358     {
  359       Label L, S;
  360       __ ldr(rscratch1, thread);
  361       __ cmp(rthread, rscratch1);
  362       __ br(Assembler::NE, S);
  363       __ get_thread(rscratch1);
  364       __ cmp(rthread, rscratch1);
  365       __ br(Assembler::EQ, L);
  366       __ BIND(S);
  367       __ stop("StubRoutines::call_stub: threads must correspond");
  368       __ BIND(L);
  369     }

  381     __ ldp(r26, r25,   r26_save);
  382     __ ldp(r24, r23,   r24_save);
  383     __ ldp(r22, r21,   r22_save);
  384     __ ldp(r20, r19,   r20_save);
  385 
  386     // restore fpcr
  387     __ ldr(rscratch1,  fpcr_save);
  388     __ set_fpcr(rscratch1);
  389 
  390     __ ldp(c_rarg0, c_rarg1,  call_wrapper);
  391     __ ldrw(c_rarg2, result_type);
  392     __ ldr(c_rarg3,  method);
  393     __ ldp(c_rarg4, c_rarg5,  entry_point);
  394     __ ldp(c_rarg6, c_rarg7,  parameter_size);
  395 
  396     // leave frame and return to caller
  397     __ leave();
  398     __ ret(lr);
  399 
  400     // handle return types different from T_INT
  401     __ BIND(check_prim);
  402     if (InlineTypeReturnedAsFields) {
  403       // Check for scalarized return value
  404       __ tbz(r0, 0, is_long);
  405       // Load pack handler address
  406       __ andr(rscratch1, r0, -2);
  407       __ ldr(rscratch1, Address(rscratch1, InstanceKlass::adr_inlineklass_fixed_block_offset()));
  408       __ ldr(rscratch1, Address(rscratch1, InlineKlass::pack_handler_jobject_offset()));
  409       __ blr(rscratch1);
  410       __ b(exit);
  411     }
  412 
  413     __ BIND(is_long);
  414     __ str(r0, Address(Rresult, 0));
  415     __ br(Assembler::AL, exit);
  416 
  417     __ BIND(is_float);
  418     __ strs(j_farg0, Address(Rresult, 0));
  419     __ br(Assembler::AL, exit);
  420 
  421     __ BIND(is_double);
  422     __ strd(j_farg0, Address(Rresult, 0));
  423     __ br(Assembler::AL, exit);
  424 
  425     return start;
  426   }
  427 
  428   // Return point for a Java call if there's an exception thrown in
  429   // Java code.  The exception is caught and transformed into a
  430   // pending exception stored in JavaThread that can be tested from
  431   // within the VM.
  432   //
  433   // Note: Usually the parameters are removed by the callee. In case
  434   // of an exception crossing an activation frame boundary, that is
  435   // not the case if the callee is compiled code => need to setup the
  436   // rsp.
  437   //
  438   // r0: exception oop
  439 
  440   address generate_catch_exception() {
  441     StubId stub_id = StubId::stubgen_catch_exception_id;
  442     StubCodeMark mark(this, stub_id);

 2223     //  |array_tag|     | header_size | element_type |     |log2_element_size|
 2224     // 32        30    24            16              8     2                 0
 2225     //
 2226     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
 2227     //
 2228 
 2229     const int lh_offset = in_bytes(Klass::layout_helper_offset());
 2230 
 2231     // Handle objArrays completely differently...
 2232     const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
 2233     __ ldrw(lh, Address(scratch_src_klass, lh_offset));
 2234     __ movw(rscratch1, objArray_lh);
 2235     __ eorw(rscratch2, lh, rscratch1);
 2236     __ cbzw(rscratch2, L_objArray);
 2237 
 2238     //  if (src->klass() != dst->klass()) return -1;
 2239     __ load_klass(rscratch2, dst);
 2240     __ eor(rscratch2, rscratch2, scratch_src_klass);
 2241     __ cbnz(rscratch2, L_failed);
 2242 
 2243     // Check for flat inline type array -> return -1
 2244     __ test_flat_array_oop(src, rscratch2, L_failed);
 2245 
 2246     // Check for null-free (non-flat) inline type array -> handle as object array
 2247     __ test_null_free_array_oop(src, rscratch2, L_objArray);
 2248 
 2249     //  if (!src->is_Array()) return -1;
 2250     __ tbz(lh, 31, L_failed);  // i.e. (lh >= 0)
 2251 
 2252     // At this point, it is known to be a typeArray (array_tag 0x3).
 2253 #ifdef ASSERT
 2254     {
 2255       BLOCK_COMMENT("assert primitive array {");
 2256       Label L;
 2257       __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
 2258       __ cmpw(lh, rscratch2);
 2259       __ br(Assembler::GE, L);
 2260       __ stop("must be a primitive array");
 2261       __ bind(L);
 2262       BLOCK_COMMENT("} assert primitive array done");
 2263     }
 2264 #endif
 2265 
 2266     arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
 2267                            rscratch2, L_failed);
 2268 

10466     gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
10467 
10468     AtomicStubMark mark_cmpxchg_4_release
10469       (_masm, &aarch64_atomic_cmpxchg_4_release_impl);
10470     gen_cas_entry(MacroAssembler::word, memory_order_release);
10471     AtomicStubMark mark_cmpxchg_8_release
10472       (_masm, &aarch64_atomic_cmpxchg_8_release_impl);
10473     gen_cas_entry(MacroAssembler::xword, memory_order_release);
10474 
10475     AtomicStubMark mark_cmpxchg_4_seq_cst
10476       (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
10477     gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
10478     AtomicStubMark mark_cmpxchg_8_seq_cst
10479       (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
10480     gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
10481 
10482     ICache::invalidate_range(first_entry, __ pc() - first_entry);
10483   }
10484 #endif // LINUX
10485 
10486   static void save_return_registers(MacroAssembler* masm) {
10487     if (InlineTypeReturnedAsFields) {
10488       masm->push(RegSet::range(r0, r7), sp);
10489       masm->sub(sp, sp, 4 * wordSize);
10490       masm->st1(v0, v1, v2, v3, masm->T1D, Address(sp));
10491       masm->sub(sp, sp, 4 * wordSize);
10492       masm->st1(v4, v5, v6, v7, masm->T1D, Address(sp));
10493     } else {
10494       masm->fmovd(rscratch1, v0);
10495       masm->stp(rscratch1, r0, Address(masm->pre(sp, -2 * wordSize)));
10496     }
10497   }
10498 
10499   static void restore_return_registers(MacroAssembler* masm) {
10500     if (InlineTypeReturnedAsFields) {
10501       masm->ld1(v4, v5, v6, v7, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
10502       masm->ld1(v0, v1, v2, v3, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
10503       masm->pop(RegSet::range(r0, r7), sp);
10504     } else {
10505       masm->ldp(rscratch1, r0, Address(masm->post(sp, 2 * wordSize)));
10506       masm->fmovd(v0, rscratch1);
10507     }
10508   }
10509 
10510   address generate_cont_thaw(Continuation::thaw_kind kind) {
10511     bool return_barrier = Continuation::is_thaw_return_barrier(kind);
10512     bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
10513 
10514     address start = __ pc();
10515 
10516     if (return_barrier) {
10517       __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
10518       __ mov(sp, rscratch1);
10519     }
10520     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
10521 
10522     if (return_barrier) {
10523       // preserve possible return value from a method returning to the return barrier
10524       save_return_registers(_masm);

10525     }
10526 
10527     __ movw(c_rarg1, (return_barrier ? 1 : 0));
10528     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
10529     __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
10530 
10531     if (return_barrier) {
10532       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
10533       restore_return_registers(_masm);

10534     }
10535     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
10536 
10537 
10538     Label thaw_success;
10539     // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
10540     __ cbnz(rscratch2, thaw_success);
10541     __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
10542     __ br(rscratch1);
10543     __ bind(thaw_success);
10544 
10545     // make room for the thawed frames
10546     __ sub(rscratch1, sp, rscratch2);
10547     __ andr(rscratch1, rscratch1, -16); // align
10548     __ mov(sp, rscratch1);
10549 
10550     if (return_barrier) {
10551       // save original return value -- again
10552       save_return_registers(_masm);

10553     }
10554 
10555     // If we want, we can templatize thaw by kind, and have three different entries
10556     __ movw(c_rarg1, (uint32_t)kind);
10557 
10558     __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
10559     __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
10560 
10561     if (return_barrier) {
10562       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
10563       restore_return_registers(_masm);

10564     } else {
10565       __ mov(r0, zr); // return 0 (success) from doYield
10566     }
10567 
10568     // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
10569     __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
10570     __ mov(rfp, sp);
10571 
10572     if (return_barrier_exception) {
10573       __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
10574       __ authenticate_return_address(c_rarg1);
10575       __ verify_oop(r0);
10576       // save return value containing the exception oop in callee-saved R19
10577       __ mov(r19, r0);
10578 
10579       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
10580 
10581       // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
10582       // __ reinitialize_ptrue();
10583 

11680     //       assert(Ra == Pa_base[j], "must be");
11681     //       MACC(Ra, Ra, t0, t1, t2);
11682     //     }
11683     //     iters =  (2*len-i)/2;
11684     //     assert(iters == len-j, "must be");
11685     //     for (; iters--; j++) {
11686     //       assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
11687     //       MACC(Rm, Rn, t0, t1, t2);
11688     //       Rm = *++Pm;
11689     //       Rn = *--Pn;
11690     //     }
11691     //     Pm_base[i-len] = t0;
11692     //     t0 = t1; t1 = t2; t2 = 0;
11693     //   }
11694 
11695     //   while (t0)
11696     //     t0 = sub(Pm_base, Pn_base, t0, len);
11697     // }
11698   };
11699 
11700   // Call here from the interpreter or compiled code to either load
11701   // multiple returned values from the inline type instance being
11702   // returned to registers or to store returned values to a newly
11703   // allocated inline type instance.
11704   address generate_return_value_stub(address destination, const char* name, bool has_res) {
11705     // We need to save all registers the calling convention may use so
11706     // the runtime calls read or update those registers. This needs to
11707     // be in sync with SharedRuntime::java_return_convention().
11708     // n.b. aarch64 asserts that frame::arg_reg_save_area_bytes == 0
11709     enum layout {
11710       j_rarg7_off = 0, j_rarg7_2,    // j_rarg7 is r0
11711       j_rarg6_off, j_rarg6_2,
11712       j_rarg5_off, j_rarg5_2,
11713       j_rarg4_off, j_rarg4_2,
11714       j_rarg3_off, j_rarg3_2,
11715       j_rarg2_off, j_rarg2_2,
11716       j_rarg1_off, j_rarg1_2,
11717       j_rarg0_off, j_rarg0_2,
11718 
11719       j_farg7_off, j_farg7_2,
11720       j_farg6_off, j_farg6_2,
11721       j_farg5_off, j_farg5_2,
11722       j_farg4_off, j_farg4_2,
11723       j_farg3_off, j_farg3_2,
11724       j_farg2_off, j_farg2_2,
11725       j_farg1_off, j_farg1_2,
11726       j_farg0_off, j_farg0_2,
11727 
11728       rfp_off, rfp_off2,
11729       return_off, return_off2,
11730 
11731       framesize // inclusive of return address
11732     };
11733 
11734     CodeBuffer code(name, 512, 64);
11735     MacroAssembler* masm = new MacroAssembler(&code);
11736 
11737     int frame_size_in_bytes = align_up(framesize*BytesPerInt, 16);
11738     assert(frame_size_in_bytes == framesize*BytesPerInt, "misaligned");
11739     int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
11740     int frame_size_in_words = frame_size_in_bytes / wordSize;
11741 
11742     OopMapSet* oop_maps = new OopMapSet();
11743     OopMap* map = new OopMap(frame_size_in_slots, 0);
11744 
11745     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg7_off), j_rarg7->as_VMReg());
11746     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg6_off), j_rarg6->as_VMReg());
11747     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg5_off), j_rarg5->as_VMReg());
11748     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg4_off), j_rarg4->as_VMReg());
11749     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg3_off), j_rarg3->as_VMReg());
11750     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg2_off), j_rarg2->as_VMReg());
11751     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg1_off), j_rarg1->as_VMReg());
11752     map->set_callee_saved(VMRegImpl::stack2reg(j_rarg0_off), j_rarg0->as_VMReg());
11753 
11754     map->set_callee_saved(VMRegImpl::stack2reg(j_farg0_off), j_farg0->as_VMReg());
11755     map->set_callee_saved(VMRegImpl::stack2reg(j_farg1_off), j_farg1->as_VMReg());
11756     map->set_callee_saved(VMRegImpl::stack2reg(j_farg2_off), j_farg2->as_VMReg());
11757     map->set_callee_saved(VMRegImpl::stack2reg(j_farg3_off), j_farg3->as_VMReg());
11758     map->set_callee_saved(VMRegImpl::stack2reg(j_farg4_off), j_farg4->as_VMReg());
11759     map->set_callee_saved(VMRegImpl::stack2reg(j_farg5_off), j_farg5->as_VMReg());
11760     map->set_callee_saved(VMRegImpl::stack2reg(j_farg6_off), j_farg6->as_VMReg());
11761     map->set_callee_saved(VMRegImpl::stack2reg(j_farg7_off), j_farg7->as_VMReg());
11762 
11763     address start = __ pc();
11764 
11765     __ enter(); // Save FP and LR before call
11766 
11767     __ stpd(j_farg1, j_farg0, Address(__ pre(sp, -2 * wordSize)));
11768     __ stpd(j_farg3, j_farg2, Address(__ pre(sp, -2 * wordSize)));
11769     __ stpd(j_farg5, j_farg4, Address(__ pre(sp, -2 * wordSize)));
11770     __ stpd(j_farg7, j_farg6, Address(__ pre(sp, -2 * wordSize)));
11771 
11772     __ stp(j_rarg1, j_rarg0, Address(__ pre(sp, -2 * wordSize)));
11773     __ stp(j_rarg3, j_rarg2, Address(__ pre(sp, -2 * wordSize)));
11774     __ stp(j_rarg5, j_rarg4, Address(__ pre(sp, -2 * wordSize)));
11775     __ stp(j_rarg7, j_rarg6, Address(__ pre(sp, -2 * wordSize)));
11776 
11777     int frame_complete = __ offset();
11778 
11779     // Set up last_Java_sp and last_Java_fp
11780     address the_pc = __ pc();
11781     __ set_last_Java_frame(sp, noreg, the_pc, rscratch1);
11782 
11783     // Call runtime
11784     __ mov(c_rarg1, r0);
11785     __ mov(c_rarg0, rthread);
11786 
11787     __ mov(rscratch1, destination);
11788     __ blr(rscratch1);
11789 
11790     oop_maps->add_gc_map(the_pc - start, map);
11791 
11792     __ reset_last_Java_frame(false);
11793 
11794     __ ldp(j_rarg7, j_rarg6, Address(__ post(sp, 2 * wordSize)));
11795     __ ldp(j_rarg5, j_rarg4, Address(__ post(sp, 2 * wordSize)));
11796     __ ldp(j_rarg3, j_rarg2, Address(__ post(sp, 2 * wordSize)));
11797     __ ldp(j_rarg1, j_rarg0, Address(__ post(sp, 2 * wordSize)));
11798 
11799     __ ldpd(j_farg7, j_farg6, Address(__ post(sp, 2 * wordSize)));
11800     __ ldpd(j_farg5, j_farg4, Address(__ post(sp, 2 * wordSize)));
11801     __ ldpd(j_farg3, j_farg2, Address(__ post(sp, 2 * wordSize)));
11802     __ ldpd(j_farg1, j_farg0, Address(__ post(sp, 2 * wordSize)));
11803 
11804     __ leave();
11805 
11806     // check for pending exceptions
11807     Label pending;
11808     __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
11809     __ cbnz(rscratch1, pending);
11810 
11811     if (has_res) {
11812       __ get_vm_result_oop(r0, rthread);
11813     }
11814 
11815     __ ret(lr);
11816 
11817     __ bind(pending);
11818     __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
11819 
11820     // -------------
11821     // make sure all code is generated
11822     masm->flush();
11823 
11824     RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, frame_size_in_words, oop_maps, false);
11825     return stub->entry_point();
11826   }
11827 
11828   // Initialization
11829   void generate_preuniverse_stubs() {
11830     // preuniverse stubs are not needed for aarch64
11831   }
11832 
11833   void generate_initial_stubs() {
11834     // Generate initial stubs and initializes the entry points
11835 
11836     // entry points that exist in all platforms Note: This is code
11837     // that could be shared among different platforms - however the
11838     // benefit seems to be smaller than the disadvantage of having a
11839     // much more complicated generator structure. See also comment in
11840     // stubRoutines.hpp.
11841 
11842     StubRoutines::_forward_exception_entry = generate_forward_exception();
11843 
11844     StubRoutines::_call_stub_entry =
11845       generate_call_stub(StubRoutines::_call_stub_return_address);
11846 
11847     // is referenced by megamorphic call

11856       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
11857     }
11858 
11859     if (UseCRC32CIntrinsics) {
11860       StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
11861     }
11862 
11863     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
11864       StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
11865     }
11866 
11867     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
11868       StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
11869     }
11870 
11871     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
11872         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
11873       StubRoutines::_hf2f = generate_float16ToFloat();
11874       StubRoutines::_f2hf = generate_floatToFloat16();
11875     }
11876 
11877     if (InlineTypeReturnedAsFields) {
11878       StubRoutines::_load_inline_type_fields_in_regs =
11879          generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::load_inline_type_fields_in_regs), "load_inline_type_fields_in_regs", false);
11880       StubRoutines::_store_inline_type_fields_to_buf =
11881          generate_return_value_stub(CAST_FROM_FN_PTR(address, SharedRuntime::store_inline_type_fields_to_buf), "store_inline_type_fields_to_buf", true);
11882     }
11883 
11884   }
11885 
11886   void generate_continuation_stubs() {
11887     // Continuation stubs:
11888     StubRoutines::_cont_thaw          = generate_cont_thaw();
11889     StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
11890     StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
11891     StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
11892   }
11893 
11894   void generate_final_stubs() {
11895     // support for verify_oop (must happen after universe_init)
11896     if (VerifyOops) {
11897       StubRoutines::_verify_oop_subroutine_entry   = generate_verify_oop();
11898     }
11899 
11900     // arraycopy stubs used by compilers
11901     generate_arraycopy_stubs();
11902 
11903     StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
< prev index next >