< prev index next >

src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp

Print this page

  31 #include "gc/shared/barrierSet.hpp"
  32 #include "gc/shared/barrierSetAssembler.hpp"
  33 #include "gc/shared/gc_globals.hpp"
  34 #include "gc/shared/tlab_globals.hpp"
  35 #include "interpreter/interpreter.hpp"
  36 #include "memory/universe.hpp"
  37 #include "nativeInst_aarch64.hpp"
  38 #include "oops/instanceOop.hpp"
  39 #include "oops/method.hpp"
  40 #include "oops/objArrayKlass.hpp"
  41 #include "oops/oop.inline.hpp"
  42 #include "prims/methodHandles.hpp"
  43 #include "runtime/atomic.hpp"
  44 #include "runtime/frame.inline.hpp"
  45 #include "runtime/handles.inline.hpp"
  46 #include "runtime/sharedRuntime.hpp"
  47 #include "runtime/stubCodeGenerator.hpp"
  48 #include "runtime/stubRoutines.hpp"
  49 #include "runtime/thread.inline.hpp"
  50 #include "utilities/align.hpp"

  51 #include "utilities/powerOfTwo.hpp"
  52 #ifdef COMPILER2
  53 #include "opto/runtime.hpp"
  54 #endif
  55 #if INCLUDE_ZGC
  56 #include "gc/z/zThreadLocalData.hpp"
  57 #endif
  58 
  59 // Declaration and definition of StubGenerator (no .hpp file).
  60 // For a more detailed description of the stub routine structure
  61 // see the comment in stubRoutines.hpp
  62 
  63 #undef __
  64 #define __ _masm->
  65 #define TIMES_OOP Address::sxtw(exact_log2(UseCompressedOops ? 4 : 8))
  66 
  67 #ifdef PRODUCT
  68 #define BLOCK_COMMENT(str) /* nothing */
  69 #else
  70 #define BLOCK_COMMENT(str) __ block_comment(str)
  71 #endif
  72 
  73 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  74 




  75 // Stub Code definitions
  76 
  77 class StubGenerator: public StubCodeGenerator {
  78  private:
  79 
  80 #ifdef PRODUCT
  81 #define inc_counter_np(counter) ((void)0)
  82 #else
  83   void inc_counter_np_(int& counter) {
  84     __ lea(rscratch2, ExternalAddress((address)&counter));
  85     __ ldrw(rscratch1, Address(rscratch2));
  86     __ addw(rscratch1, rscratch1, 1);
  87     __ strw(rscratch1, Address(rscratch2));
  88   }
  89 #define inc_counter_np(counter) \
  90   BLOCK_COMMENT("inc_counter " #counter); \
  91   inc_counter_np_(counter);
  92 #endif
  93 
  94   // Call stubs are used to call Java from C

 331 
 332     // pop parameters
 333     __ sub(esp, rfp, -sp_after_call_off * wordSize);
 334 
 335 #ifdef ASSERT
 336     // verify that threads correspond
 337     {
 338       Label L, S;
 339       __ ldr(rscratch1, thread);
 340       __ cmp(rthread, rscratch1);
 341       __ br(Assembler::NE, S);
 342       __ get_thread(rscratch1);
 343       __ cmp(rthread, rscratch1);
 344       __ br(Assembler::EQ, L);
 345       __ BIND(S);
 346       __ stop("StubRoutines::call_stub: threads must correspond");
 347       __ BIND(L);
 348     }
 349 #endif
 350 


 351     // restore callee-save registers
 352     __ ldpd(v15, v14,  d15_save);
 353     __ ldpd(v13, v12,  d13_save);
 354     __ ldpd(v11, v10,  d11_save);
 355     __ ldpd(v9,  v8,   d9_save);
 356 
 357     __ ldp(r28, r27,   r28_save);
 358     __ ldp(r26, r25,   r26_save);
 359     __ ldp(r24, r23,   r24_save);
 360     __ ldp(r22, r21,   r22_save);
 361     __ ldp(r20, r19,   r20_save);
 362 
 363     __ ldp(c_rarg0, c_rarg1,  call_wrapper);
 364     __ ldrw(c_rarg2, result_type);
 365     __ ldr(c_rarg3,  method);
 366     __ ldp(c_rarg4, c_rarg5,  entry_point);
 367     __ ldp(c_rarg6, c_rarg7,  parameter_size);
 368 
 369     // leave frame and return to caller
 370     __ leave();

6586     gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
6587 
6588     AtomicStubMark mark_cmpxchg_4_release
6589       (_masm, &aarch64_atomic_cmpxchg_4_release_impl);
6590     gen_cas_entry(MacroAssembler::word, memory_order_release);
6591     AtomicStubMark mark_cmpxchg_8_release
6592       (_masm, &aarch64_atomic_cmpxchg_8_release_impl);
6593     gen_cas_entry(MacroAssembler::xword, memory_order_release);
6594 
6595     AtomicStubMark mark_cmpxchg_4_seq_cst
6596       (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
6597     gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
6598     AtomicStubMark mark_cmpxchg_8_seq_cst
6599       (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
6600     gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
6601 
6602     ICache::invalidate_range(first_entry, __ pc() - first_entry);
6603   }
6604 #endif // LINUX
6605 







































































































































































































































































































6606   // Continuation point for throwing of implicit exceptions that are
6607   // not handled in the current activation. Fabricates an exception
6608   // oop and initiates normal exception dispatching in this
6609   // frame. Since we need to preserve callee-saved values (currently
6610   // only for C2, but done for C1 as well) we need a callee-saved oop
6611   // map and therefore have to make these stubs into RuntimeStubs
6612   // rather than BufferBlobs.  If the compiler needs all registers to
6613   // be preserved between the fault point and the exception handler
6614   // then it must assume responsibility for that in
6615   // AbstractCompiler::continuation_for_implicit_null_exception or
6616   // continuation_for_implicit_division_by_zero_exception. All other
6617   // implicit exceptions (e.g., NullPointerException or
6618   // AbstractMethodError on entry) are either at call sites or
6619   // otherwise assume that stack unwinding will be initiated, so
6620   // caller saved registers were assumed volatile in the compiler.
6621 
6622 #undef __
6623 #define __ masm->
6624 
6625   address generate_throw_exception(const char* name,

7578     //   StubRoutines::_dlog = generate_dlog();
7579     // }
7580 
7581     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
7582       StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
7583     }
7584 
7585     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
7586       StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
7587     }
7588 
7589     // Safefetch stubs.
7590     generate_safefetch("SafeFetch32", sizeof(int),     &StubRoutines::_safefetch32_entry,
7591                                                        &StubRoutines::_safefetch32_fault_pc,
7592                                                        &StubRoutines::_safefetch32_continuation_pc);
7593     generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
7594                                                        &StubRoutines::_safefetchN_fault_pc,
7595                                                        &StubRoutines::_safefetchN_continuation_pc);
7596   }
7597 














7598   void generate_all() {
7599     // support for verify_oop (must happen after universe_init)
7600     StubRoutines::_verify_oop_subroutine_entry     = generate_verify_oop();
7601     StubRoutines::_throw_AbstractMethodError_entry =
7602       generate_throw_exception("AbstractMethodError throw_exception",
7603                                CAST_FROM_FN_PTR(address,
7604                                                 SharedRuntime::
7605                                                 throw_AbstractMethodError));
7606 
7607     StubRoutines::_throw_IncompatibleClassChangeError_entry =
7608       generate_throw_exception("IncompatibleClassChangeError throw_exception",
7609                                CAST_FROM_FN_PTR(address,
7610                                                 SharedRuntime::
7611                                                 throw_IncompatibleClassChangeError));
7612 
7613     StubRoutines::_throw_NullPointerException_at_call_entry =
7614       generate_throw_exception("NullPointerException at call throw_exception",
7615                                CAST_FROM_FN_PTR(address,
7616                                                 SharedRuntime::
7617                                                 throw_NullPointerException_at_call));

7718       StubRoutines::_sha3_implCompressMB   = generate_sha3_implCompress(true,    "sha3_implCompressMB");
7719     }
7720 
7721     // generate Adler32 intrinsics code
7722     if (UseAdler32Intrinsics) {
7723       StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32();
7724     }
7725 
7726     StubRoutines::aarch64::_spin_wait = generate_spin_wait();
7727 
7728 #ifdef LINUX
7729 
7730     generate_atomic_entry_points();
7731 
7732 #endif // LINUX
7733 
7734     StubRoutines::aarch64::set_completed();
7735   }
7736 
7737  public:
7738   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
7739     if (all) {
7740       generate_all();
7741     } else {
7742       generate_initial();




7743     }
7744   }
7745 }; // end class declaration
7746 
7747 #define UCM_TABLE_MAX_ENTRIES 8
7748 void StubGenerator_generate(CodeBuffer* code, bool all) {
7749   if (UnsafeCopyMemory::_table == NULL) {
7750     UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES);
7751   }
7752   StubGenerator g(code, all);
7753 }
7754 
7755 
7756 #ifdef LINUX
7757 
7758 // Define pointers to atomic stubs and initialize them to point to the
7759 // code in atomic_aarch64.S.
7760 
7761 #define DEFAULT_ATOMIC_OP(OPNAME, SIZE, RELAXED)                                \
7762   extern "C" uint64_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl \
7763     (volatile void *ptr, uint64_t arg1, uint64_t arg2);                 \
7764   aarch64_atomic_stub_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _impl \
7765     = aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl;
7766 
7767 DEFAULT_ATOMIC_OP(fetch_add, 4, )
7768 DEFAULT_ATOMIC_OP(fetch_add, 8, )
7769 DEFAULT_ATOMIC_OP(fetch_add, 4, _relaxed)
7770 DEFAULT_ATOMIC_OP(fetch_add, 8, _relaxed)
7771 DEFAULT_ATOMIC_OP(xchg, 4, )
7772 DEFAULT_ATOMIC_OP(xchg, 8, )
7773 DEFAULT_ATOMIC_OP(cmpxchg, 1, )
7774 DEFAULT_ATOMIC_OP(cmpxchg, 4, )
7775 DEFAULT_ATOMIC_OP(cmpxchg, 8, )
7776 DEFAULT_ATOMIC_OP(cmpxchg, 1, _relaxed)
7777 DEFAULT_ATOMIC_OP(cmpxchg, 4, _relaxed)
7778 DEFAULT_ATOMIC_OP(cmpxchg, 8, _relaxed)
7779 DEFAULT_ATOMIC_OP(cmpxchg, 4, _release)
7780 DEFAULT_ATOMIC_OP(cmpxchg, 8, _release)
7781 DEFAULT_ATOMIC_OP(cmpxchg, 4, _seq_cst)
7782 DEFAULT_ATOMIC_OP(cmpxchg, 8, _seq_cst)
7783 
7784 #undef DEFAULT_ATOMIC_OP
7785 
7786 #endif // LINUX






































































  31 #include "gc/shared/barrierSet.hpp"
  32 #include "gc/shared/barrierSetAssembler.hpp"
  33 #include "gc/shared/gc_globals.hpp"
  34 #include "gc/shared/tlab_globals.hpp"
  35 #include "interpreter/interpreter.hpp"
  36 #include "memory/universe.hpp"
  37 #include "nativeInst_aarch64.hpp"
  38 #include "oops/instanceOop.hpp"
  39 #include "oops/method.hpp"
  40 #include "oops/objArrayKlass.hpp"
  41 #include "oops/oop.inline.hpp"
  42 #include "prims/methodHandles.hpp"
  43 #include "runtime/atomic.hpp"
  44 #include "runtime/frame.inline.hpp"
  45 #include "runtime/handles.inline.hpp"
  46 #include "runtime/sharedRuntime.hpp"
  47 #include "runtime/stubCodeGenerator.hpp"
  48 #include "runtime/stubRoutines.hpp"
  49 #include "runtime/thread.inline.hpp"
  50 #include "utilities/align.hpp"
  51 #include "utilities/globalDefinitions.hpp"
  52 #include "utilities/powerOfTwo.hpp"
  53 #ifdef COMPILER2
  54 #include "opto/runtime.hpp"
  55 #endif
  56 #if INCLUDE_ZGC
  57 #include "gc/z/zThreadLocalData.hpp"
  58 #endif
  59 
  60 // Declaration and definition of StubGenerator (no .hpp file).
  61 // For a more detailed description of the stub routine structure
  62 // see the comment in stubRoutines.hpp
  63 
  64 #undef __
  65 #define __ _masm->
  66 #define TIMES_OOP Address::sxtw(exact_log2(UseCompressedOops ? 4 : 8))
  67 
  68 #ifdef PRODUCT
  69 #define BLOCK_COMMENT(str) /* nothing */
  70 #else
  71 #define BLOCK_COMMENT(str) __ block_comment(str)
  72 #endif
  73 
  74 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  75 
  76 OopMap* continuation_enter_setup(MacroAssembler* masm, int& stack_slots);
  77 void fill_continuation_entry(MacroAssembler* masm);
  78 void continuation_enter_cleanup(MacroAssembler* masm);
  79 
  80 // Stub Code definitions
  81 
  82 class StubGenerator: public StubCodeGenerator {
  83  private:
  84 
  85 #ifdef PRODUCT
  86 #define inc_counter_np(counter) ((void)0)
  87 #else
  88   void inc_counter_np_(int& counter) {
  89     __ lea(rscratch2, ExternalAddress((address)&counter));
  90     __ ldrw(rscratch1, Address(rscratch2));
  91     __ addw(rscratch1, rscratch1, 1);
  92     __ strw(rscratch1, Address(rscratch2));
  93   }
  94 #define inc_counter_np(counter) \
  95   BLOCK_COMMENT("inc_counter " #counter); \
  96   inc_counter_np_(counter);
  97 #endif
  98 
  99   // Call stubs are used to call Java from C

 336 
 337     // pop parameters
 338     __ sub(esp, rfp, -sp_after_call_off * wordSize);
 339 
 340 #ifdef ASSERT
 341     // verify that threads correspond
 342     {
 343       Label L, S;
 344       __ ldr(rscratch1, thread);
 345       __ cmp(rthread, rscratch1);
 346       __ br(Assembler::NE, S);
 347       __ get_thread(rscratch1);
 348       __ cmp(rthread, rscratch1);
 349       __ br(Assembler::EQ, L);
 350       __ BIND(S);
 351       __ stop("StubRoutines::call_stub: threads must correspond");
 352       __ BIND(L);
 353     }
 354 #endif
 355 
 356     __ pop_cont_fastpath(rthread);
 357 
 358     // restore callee-save registers
 359     __ ldpd(v15, v14,  d15_save);
 360     __ ldpd(v13, v12,  d13_save);
 361     __ ldpd(v11, v10,  d11_save);
 362     __ ldpd(v9,  v8,   d9_save);
 363 
 364     __ ldp(r28, r27,   r28_save);
 365     __ ldp(r26, r25,   r26_save);
 366     __ ldp(r24, r23,   r24_save);
 367     __ ldp(r22, r21,   r22_save);
 368     __ ldp(r20, r19,   r20_save);
 369 
 370     __ ldp(c_rarg0, c_rarg1,  call_wrapper);
 371     __ ldrw(c_rarg2, result_type);
 372     __ ldr(c_rarg3,  method);
 373     __ ldp(c_rarg4, c_rarg5,  entry_point);
 374     __ ldp(c_rarg6, c_rarg7,  parameter_size);
 375 
 376     // leave frame and return to caller
 377     __ leave();

6593     gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
6594 
6595     AtomicStubMark mark_cmpxchg_4_release
6596       (_masm, &aarch64_atomic_cmpxchg_4_release_impl);
6597     gen_cas_entry(MacroAssembler::word, memory_order_release);
6598     AtomicStubMark mark_cmpxchg_8_release
6599       (_masm, &aarch64_atomic_cmpxchg_8_release_impl);
6600     gen_cas_entry(MacroAssembler::xword, memory_order_release);
6601 
6602     AtomicStubMark mark_cmpxchg_4_seq_cst
6603       (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
6604     gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
6605     AtomicStubMark mark_cmpxchg_8_seq_cst
6606       (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
6607     gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
6608 
6609     ICache::invalidate_range(first_entry, __ pc() - first_entry);
6610   }
6611 #endif // LINUX
6612 
6613 RuntimeStub* generate_cont_doYield() {
6614     const char *name = "cont_doYield";
6615 
6616     enum layout {
6617       rfp_off1,
6618       rfp_off2,
6619       lr_off,
6620       lr_off2,
6621       framesize // inclusive of return address
6622     };
6623     // assert(is_even(framesize/2), "sp not 16-byte aligned");
6624     
6625     int insts_size = 512;
6626     int locs_size  = 64;
6627     CodeBuffer code(name, insts_size, locs_size);
6628     OopMapSet* oop_maps  = new OopMapSet();
6629     MacroAssembler* masm = new MacroAssembler(&code);
6630     MacroAssembler* _masm = masm;
6631 
6632     address start = __ pc();
6633 
6634     __ enter();
6635 
6636     __ mov(c_rarg1, sp);
6637 
6638     int frame_complete = __ pc() - start;
6639     address the_pc = __ pc();
6640 
6641     __ post_call_nop(); // this must be exactly after the pc value that is pushed into the frame info, we use this nop for fast CodeBlob lookup
6642 
6643     __ mov(c_rarg0, rthread);
6644     __ set_last_Java_frame(sp, rfp, the_pc, rscratch1);
6645     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::freeze), 2);
6646     __ reset_last_Java_frame(true);
6647 
6648     Label pinned;
6649 
6650     __ cbnz(r0, pinned);
6651 
6652     __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
6653     __ mov(sp, rscratch1);
6654     continuation_enter_cleanup(masm);
6655 
6656     __ bind(pinned); // pinned -- return to caller
6657     
6658     __ leave();
6659     __ ret(lr);
6660 
6661     OopMap* map = new OopMap(framesize, 1);
6662     // map->set_callee_saved(VMRegImpl::stack2reg(rfp_off), rfp->as_VMReg());
6663     oop_maps->add_gc_map(the_pc - start, map);
6664 
6665     RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size)
6666     RuntimeStub::new_runtime_stub(name,
6667                                   &code,
6668                                   frame_complete,
6669                                   (framesize >> (LogBytesPerWord - LogBytesPerInt)),
6670                                   oop_maps, false);
6671     return stub;
6672   }
6673 
6674   address generate_cont_jump_from_safepoint() {
6675     __ align(CodeEntryAlignment);
6676     StubCodeMark mark(this, "StubRoutines","Continuation jump from safepoint");
6677 
6678     address start = __ pc();
6679 
6680 #ifdef ASSERT
6681     { // verify that threads correspond
6682       Label L;
6683       __ get_thread(rscratch1);
6684       __ cmp(rthread, rscratch1);
6685       __ br(Assembler::EQ, L);
6686       __ stop("StubRoutines::cont_jump_from_safepoint: threads must correspond");
6687       __ BIND(L);
6688     }
6689 #endif
6690 
6691     __ reset_last_Java_frame(true); // false would be fine, too, I guess
6692     __ reinit_heapbase();
6693     
6694     __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
6695     __ mov(sp, rscratch1);
6696     continuation_enter_cleanup(_masm);
6697     __ leave();
6698     __ ret(lr);
6699 
6700     return start;
6701   }
6702 
6703   address generate_cont_thaw(bool return_barrier, bool exception) {
6704     assert (return_barrier || !exception, "must be");
6705 
6706     address start = __ pc();
6707 
6708     if (return_barrier) {
6709       __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
6710       __ mov(sp, rscratch1);
6711     }
6712     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
6713 
6714     if (return_barrier) {
6715       // preserve possible return value from a method returning to the return barrier
6716       __ fmovd(rscratch1, v0);
6717       __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
6718     }
6719 
6720     __ movw(c_rarg1, (return_barrier ? 1 : 0) + (exception ? 1 : 0));
6721     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
6722     __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
6723 
6724     if (return_barrier) {
6725       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
6726       __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
6727       __ fmovd(v0, rscratch1);
6728     }
6729     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
6730 
6731 
6732     Label thaw_success;
6733     // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
6734     __ cbnz(rscratch2, thaw_success);
6735     __ lea(rscratch1, ExternalAddress(StubRoutines::throw_StackOverflowError_entry()));
6736     __ br(rscratch1);
6737     __ bind(thaw_success);
6738     
6739     // make room for the thawed frames
6740     __ sub(rscratch1, sp, rscratch2);
6741     __ andr(rscratch1, rscratch1, -16); // align
6742     __ mov(sp, rscratch1);
6743     
6744     if (return_barrier) {
6745       // save original return value -- again
6746       __ fmovd(rscratch1, v0);
6747       __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
6748     }
6749 
6750     __ movw(c_rarg1, (return_barrier ? 1 : 0) + (exception ? 1 : 0));
6751     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::thaw), rthread, c_rarg1);
6752     __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
6753 
6754     if (return_barrier) {
6755       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
6756       __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
6757       __ fmovd(v0, rscratch1);
6758     } else {
6759       __ mov(r0, zr); // return 0 (success) from doYield
6760     }
6761 
6762     // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
6763     __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
6764     __ mov(rfp, sp);
6765 
6766     if (exception) {
6767       __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
6768       __ verify_oop(r0);
6769       __ mov(r19, r0); // save return value contaning the exception oop in callee-saved R19
6770 
6771       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
6772 
6773       // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
6774       // __ reinitialize_ptrue();
6775 
6776       // see OptoRuntime::generate_exception_blob: r0 -- exception oop, r3 -- exception pc
6777 
6778       __ mov(r1, r0); // the exception handler
6779       __ mov(r0, r19); // restore return value contaning the exception oop
6780       __ verify_oop(r0);
6781 
6782       __ leave();
6783       __ mov(r3, lr);
6784       __ br(r1); // the exception handler
6785     }
6786 
6787     // We're "returning" into the topmost thawed frame; see Thaw::push_return_frame
6788     __ leave();
6789     __ ret(lr);
6790 
6791     return start;
6792   }
6793 
6794   address generate_cont_thaw() {
6795     StubCodeMark mark(this, "StubRoutines", "Cont thaw");
6796     address start = __ pc();
6797     generate_cont_thaw(false, false);
6798     return start;
6799   }
6800 
6801   address generate_cont_returnBarrier() {
6802     // TODO: will probably need multiple return barriers depending on return type
6803     StubCodeMark mark(this, "StubRoutines", "cont return barrier");
6804     address start = __ pc();
6805 
6806     generate_cont_thaw(true, false);
6807 
6808     return start;
6809   }
6810 
6811   address generate_cont_returnBarrier_exception() {
6812     StubCodeMark mark(this, "StubRoutines", "cont return barrier exception handler");
6813     address start = __ pc();
6814 
6815     generate_cont_thaw(true, true);
6816 
6817     return start;
6818   }
6819 
6820   address generate_cont_interpreter_forced_preempt_return() {
6821       StubCodeMark mark(this, "StubRoutines", "cont interpreter forced preempt return");
6822       address start = __ pc();
6823 
6824       // This is necessary for forced yields, as the return addres (in rbx) is captured in a call_VM, and skips the restoration of rbcp and locals
6825 
6826       assert_asm(_masm, __ cmp(sp, rfp), Assembler::EQ, "sp != fp"); // __ mov(rfp, sp);
6827       __ leave(); // we're now on the last thawed frame
6828 
6829       __ ldr(rbcp,    Address(rfp, frame::interpreter_frame_bcp_offset    * wordSize)); // InterpreterMacroAssembler::restore_bcp()
6830       __ ldr(rlocals, Address(rfp, frame::interpreter_frame_locals_offset * wordSize)); // InterpreterMacroAssembler::restore_locals()
6831       __ ldr(rcpool,  Address(rfp, frame::interpreter_frame_cache_offset  * wordSize)); // InterpreterMacroAssembler::restore_constant_pool_cache()
6832       __ ldr(rmethod, Address(rfp, frame::interpreter_frame_method_offset * wordSize)); // InterpreterMacroAssembler::get_method(rmethod) -- might not be necessary
6833       // __ reinit_heapbase();
6834 
6835       // Restore stack bottom in case i2c adjusted stack and NULL it as marker that esp is now tos until next java call
6836       __ ldr(esp, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
6837       __ str(zr,  Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
6838 
6839       __ ret(lr);
6840 
6841       return start;
6842     }
6843 
6844 #if INCLUDE_JFR
6845 
6846   static void jfr_set_last_java_frame(MacroAssembler* _masm, Register thread) {
6847     Register last_java_pc = c_rarg0;
6848     Register last_java_sp = c_rarg2;
6849     // __ vzeroupper();
6850     __ mov(last_java_pc, lr);
6851     __ mov(last_java_sp, sp);
6852     __ str(last_java_pc, Address(thread, JavaThread::last_Java_pc_offset()));
6853     __ str(last_java_sp, Address(thread, JavaThread::last_Java_sp_offset()));
6854   }
6855 
6856   // Must be called before enter()
6857   static void jfr_prologue(MacroAssembler* _masm, Register thread) {
6858     jfr_set_last_java_frame(_masm, thread);
6859     __ mov(c_rarg0, rthread);
6860   }
6861 
6862   // Handle is dereference here using correct load constructs.
6863   static void jfr_epilogue(MacroAssembler* _masm, Register thread) {
6864     __ reset_last_Java_frame(false);
6865     Label null_jobject;
6866     __ cbz(r0, null_jobject);
6867     DecoratorSet decorators = ACCESS_READ | IN_NATIVE;
6868     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
6869     bs->load_at(_masm, decorators, T_OBJECT, r0, Address(r0, 0), rscratch1, rthread);
6870     __ bind(null_jobject);
6871   }
6872 
6873   // For c2: c_rarg0 is junk, c_rarg1 is the thread id. Call to runtime to write a checkpoint.
6874   // Runtime will return a jobject handle to the event writer. The handle is dereferenced and the return value
6875   // is the event writer oop.
6876   address generate_jfr_write_checkpoint() {
6877     StubCodeMark mark(this, "jfr_write_checkpoint", "JFR C2 support for Virtual Threads");
6878     address start = __ pc();
6879 
6880     jfr_prologue(_masm, rthread);
6881     __ enter();
6882     __ call_VM_leaf(CAST_FROM_FN_PTR(address, JFR_WRITE_CHECKPOINT_FUNCTION), 2);
6883     jfr_epilogue(_masm, rthread);
6884     __ leave();
6885     __ ret(lr);
6886 
6887     return start;
6888   }
6889 
6890   // For c1: call the corresponding runtime routine, it returns a jobject handle to the event writer.
6891   // The handle is dereferenced and the return value is the event writer oop.
6892   address generate_jfr_get_event_writer() {
6893     StubCodeMark mark(this, "jfr_get_event_writer", "JFR C1 support for Virtual Threads");
6894     address start = __ pc();
6895 
6896     jfr_prologue(_masm, rthread);
6897     __ enter();
6898     __ call_VM_leaf(CAST_FROM_FN_PTR(address, JFR_GET_EVENT_WRITER_FUNCTION), 1);
6899     jfr_epilogue(_masm, rthread);
6900     __ leave();
6901     __ ret(lr);
6902 
6903     return start;
6904   }
6905 
6906 #endif // INCLUDE_JFR
6907 
6908   // Continuation point for throwing of implicit exceptions that are
6909   // not handled in the current activation. Fabricates an exception
6910   // oop and initiates normal exception dispatching in this
6911   // frame. Since we need to preserve callee-saved values (currently
6912   // only for C2, but done for C1 as well) we need a callee-saved oop
6913   // map and therefore have to make these stubs into RuntimeStubs
6914   // rather than BufferBlobs.  If the compiler needs all registers to
6915   // be preserved between the fault point and the exception handler
6916   // then it must assume responsibility for that in
6917   // AbstractCompiler::continuation_for_implicit_null_exception or
6918   // continuation_for_implicit_division_by_zero_exception. All other
6919   // implicit exceptions (e.g., NullPointerException or
6920   // AbstractMethodError on entry) are either at call sites or
6921   // otherwise assume that stack unwinding will be initiated, so
6922   // caller saved registers were assumed volatile in the compiler.
6923 
6924 #undef __
6925 #define __ masm->
6926 
6927   address generate_throw_exception(const char* name,

7880     //   StubRoutines::_dlog = generate_dlog();
7881     // }
7882 
7883     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
7884       StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
7885     }
7886 
7887     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
7888       StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
7889     }
7890 
7891     // Safefetch stubs.
7892     generate_safefetch("SafeFetch32", sizeof(int),     &StubRoutines::_safefetch32_entry,
7893                                                        &StubRoutines::_safefetch32_fault_pc,
7894                                                        &StubRoutines::_safefetch32_continuation_pc);
7895     generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
7896                                                        &StubRoutines::_safefetchN_fault_pc,
7897                                                        &StubRoutines::_safefetchN_continuation_pc);
7898   }
7899 
7900   void generate_phase1() {
7901     // Continuation stubs:
7902     StubRoutines::_cont_thaw          = generate_cont_thaw();
7903     StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
7904     StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
7905     StubRoutines::_cont_doYield_stub = generate_cont_doYield();
7906     StubRoutines::_cont_doYield    = StubRoutines::_cont_doYield_stub->entry_point();
7907     StubRoutines::_cont_jump_from_sp = generate_cont_jump_from_safepoint();
7908     StubRoutines::_cont_interpreter_forced_preempt_return = generate_cont_interpreter_forced_preempt_return();
7909 
7910     JFR_ONLY(StubRoutines::_jfr_write_checkpoint = generate_jfr_write_checkpoint();)
7911     JFR_ONLY(StubRoutines::_jfr_get_event_writer = generate_jfr_get_event_writer();)
7912   }
7913 
7914   void generate_all() {
7915     // support for verify_oop (must happen after universe_init)
7916     StubRoutines::_verify_oop_subroutine_entry     = generate_verify_oop();
7917     StubRoutines::_throw_AbstractMethodError_entry =
7918       generate_throw_exception("AbstractMethodError throw_exception",
7919                                CAST_FROM_FN_PTR(address,
7920                                                 SharedRuntime::
7921                                                 throw_AbstractMethodError));
7922 
7923     StubRoutines::_throw_IncompatibleClassChangeError_entry =
7924       generate_throw_exception("IncompatibleClassChangeError throw_exception",
7925                                CAST_FROM_FN_PTR(address,
7926                                                 SharedRuntime::
7927                                                 throw_IncompatibleClassChangeError));
7928 
7929     StubRoutines::_throw_NullPointerException_at_call_entry =
7930       generate_throw_exception("NullPointerException at call throw_exception",
7931                                CAST_FROM_FN_PTR(address,
7932                                                 SharedRuntime::
7933                                                 throw_NullPointerException_at_call));

8034       StubRoutines::_sha3_implCompressMB   = generate_sha3_implCompress(true,    "sha3_implCompressMB");
8035     }
8036 
8037     // generate Adler32 intrinsics code
8038     if (UseAdler32Intrinsics) {
8039       StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32();
8040     }
8041 
8042     StubRoutines::aarch64::_spin_wait = generate_spin_wait();
8043 
8044 #ifdef LINUX
8045 
8046     generate_atomic_entry_points();
8047 
8048 #endif // LINUX
8049 
8050     StubRoutines::aarch64::set_completed();
8051   }
8052 
8053  public:
8054   StubGenerator(CodeBuffer* code, int phase) : StubCodeGenerator(code) {
8055     if (phase == 0) {


8056       generate_initial();
8057     } else if (phase == 1) {
8058       generate_phase1(); // stubs that must be available for the interpreter
8059     } else {
8060       generate_all();
8061     }
8062   }
8063 }; // end class declaration
8064 
8065 #define UCM_TABLE_MAX_ENTRIES 8
8066 void StubGenerator_generate(CodeBuffer* code, int phase) {
8067   if (UnsafeCopyMemory::_table == NULL) {
8068     UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES);
8069   }
8070   StubGenerator g(code, phase);
8071 }
8072 
8073 
8074 #ifdef LINUX
8075 
8076 // Define pointers to atomic stubs and initialize them to point to the
8077 // code in atomic_aarch64.S.
8078 
8079 #define DEFAULT_ATOMIC_OP(OPNAME, SIZE, RELAXED)                                \
8080   extern "C" uint64_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl \
8081     (volatile void *ptr, uint64_t arg1, uint64_t arg2);                 \
8082   aarch64_atomic_stub_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _impl \
8083     = aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl;
8084 
8085 DEFAULT_ATOMIC_OP(fetch_add, 4, )
8086 DEFAULT_ATOMIC_OP(fetch_add, 8, )
8087 DEFAULT_ATOMIC_OP(fetch_add, 4, _relaxed)
8088 DEFAULT_ATOMIC_OP(fetch_add, 8, _relaxed)
8089 DEFAULT_ATOMIC_OP(xchg, 4, )
8090 DEFAULT_ATOMIC_OP(xchg, 8, )
8091 DEFAULT_ATOMIC_OP(cmpxchg, 1, )
8092 DEFAULT_ATOMIC_OP(cmpxchg, 4, )
8093 DEFAULT_ATOMIC_OP(cmpxchg, 8, )
8094 DEFAULT_ATOMIC_OP(cmpxchg, 1, _relaxed)
8095 DEFAULT_ATOMIC_OP(cmpxchg, 4, _relaxed)
8096 DEFAULT_ATOMIC_OP(cmpxchg, 8, _relaxed)
8097 DEFAULT_ATOMIC_OP(cmpxchg, 4, _release)
8098 DEFAULT_ATOMIC_OP(cmpxchg, 8, _release)
8099 DEFAULT_ATOMIC_OP(cmpxchg, 4, _seq_cst)
8100 DEFAULT_ATOMIC_OP(cmpxchg, 8, _seq_cst)
8101 
8102 #undef DEFAULT_ATOMIC_OP
8103 
8104 #endif // LINUX
8105 
8106 
8107 #undef __
8108 #define __ masm->
8109 
8110 // on exit, sp points to the ContinuationEntry
8111 OopMap* continuation_enter_setup(MacroAssembler* masm, int& stack_slots) {
8112   assert (ContinuationEntry::size() % VMRegImpl::stack_slot_size == 0, "");
8113   assert (in_bytes(ContinuationEntry::cont_offset())  % VMRegImpl::stack_slot_size == 0, "");
8114   assert (in_bytes(ContinuationEntry::chunk_offset()) % VMRegImpl::stack_slot_size == 0, "");
8115 
8116   stack_slots += (int)ContinuationEntry::size()/wordSize;
8117   __ sub(sp, sp, (int)ContinuationEntry::size()); // place Continuation metadata
8118 
8119   OopMap* map = new OopMap(((int)ContinuationEntry::size() + wordSize)/ VMRegImpl::stack_slot_size, 0 /* arg_slots*/);
8120   ContinuationEntry::setup_oopmap(map);
8121 
8122   __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
8123   __ str(rscratch1, Address(sp, ContinuationEntry::parent_offset()));
8124   __ mov(rscratch1, sp); // we can't use sp as the source in str
8125   __ str(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
8126 
8127   return map;
8128 }
8129 
8130 // on entry c_rarg1 points to the continuation 
8131 //          sp points to ContinuationEntry
8132 void fill_continuation_entry(MacroAssembler* masm) {
8133 #ifdef ASSERT
8134   __ movw(rscratch1, 0x1234);
8135   __ strw(rscratch1, Address(sp, ContinuationEntry::cookie_offset()));
8136 #endif
8137 
8138   __ str(c_rarg1, Address(sp, ContinuationEntry::cont_offset()));
8139   __ str(zr, Address(sp, ContinuationEntry::chunk_offset()));
8140   __ strw(zr, Address(sp, ContinuationEntry::argsize_offset()));
8141 
8142   __ ldr(rscratch1, Address(rthread, JavaThread::cont_fastpath_offset()));
8143   __ str(rscratch1, Address(sp, ContinuationEntry::parent_cont_fastpath_offset()));
8144   __ ldr(rscratch1, Address(rthread, JavaThread::held_monitor_count_offset()));
8145   __ str(rscratch1, Address(sp, ContinuationEntry::parent_held_monitor_count_offset()));
8146   
8147   __ str(zr, Address(rthread, JavaThread::cont_fastpath_offset()));
8148   __ reset_held_monitor_count(rthread);
8149 }
8150 
8151 // on entry, sp points to the ContinuationEntry
8152 // on exit, rfp points to the spilled rfp in the entry frame
8153 void continuation_enter_cleanup(MacroAssembler* masm) {
8154 #ifndef PRODUCT
8155   Label OK;
8156   __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
8157   __ cmp(sp, rscratch1);
8158   __ br(Assembler::EQ, OK);
8159   __ stop("incorrect sp1");
8160   __ bind(OK);
8161 #endif
8162   
8163   __ ldr(rscratch1, Address(sp, ContinuationEntry::parent_cont_fastpath_offset()));
8164   __ str(rscratch1, Address(rthread, JavaThread::cont_fastpath_offset()));
8165   __ ldr(rscratch1, Address(sp, ContinuationEntry::parent_held_monitor_count_offset()));
8166   __ str(rscratch1, Address(rthread, JavaThread::held_monitor_count_offset()));
8167 
8168   __ ldr(rscratch2, Address(sp, ContinuationEntry::parent_offset()));
8169   __ str(rscratch2, Address(rthread, JavaThread::cont_entry_offset()));
8170   __ add(rfp, sp, (int)ContinuationEntry::size());
8171 }
8172 
8173 #undef __
< prev index next >