< prev index next >

src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp

Print this page

  31 #include "gc/shared/barrierSet.hpp"
  32 #include "gc/shared/barrierSetAssembler.hpp"
  33 #include "gc/shared/gc_globals.hpp"
  34 #include "gc/shared/tlab_globals.hpp"
  35 #include "interpreter/interpreter.hpp"
  36 #include "memory/universe.hpp"
  37 #include "nativeInst_aarch64.hpp"
  38 #include "oops/instanceOop.hpp"
  39 #include "oops/method.hpp"
  40 #include "oops/objArrayKlass.hpp"
  41 #include "oops/oop.inline.hpp"
  42 #include "prims/methodHandles.hpp"
  43 #include "runtime/atomic.hpp"
  44 #include "runtime/frame.inline.hpp"
  45 #include "runtime/handles.inline.hpp"
  46 #include "runtime/sharedRuntime.hpp"
  47 #include "runtime/stubCodeGenerator.hpp"
  48 #include "runtime/stubRoutines.hpp"
  49 #include "runtime/thread.inline.hpp"
  50 #include "utilities/align.hpp"

  51 #include "utilities/powerOfTwo.hpp"
  52 #ifdef COMPILER2
  53 #include "opto/runtime.hpp"
  54 #endif
  55 #if INCLUDE_ZGC
  56 #include "gc/z/zThreadLocalData.hpp"
  57 #endif
  58 
  59 // Declaration and definition of StubGenerator (no .hpp file).
  60 // For a more detailed description of the stub routine structure
  61 // see the comment in stubRoutines.hpp
  62 
  63 #undef __
  64 #define __ _masm->
  65 #define TIMES_OOP Address::sxtw(exact_log2(UseCompressedOops ? 4 : 8))
  66 
  67 #ifdef PRODUCT
  68 #define BLOCK_COMMENT(str) /* nothing */
  69 #else
  70 #define BLOCK_COMMENT(str) __ block_comment(str)
  71 #endif
  72 
  73 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  74 




  75 // Stub Code definitions
  76 
  77 class StubGenerator: public StubCodeGenerator {
  78  private:
  79 
  80 #ifdef PRODUCT
  81 #define inc_counter_np(counter) ((void)0)
  82 #else
  83   void inc_counter_np_(int& counter) {
  84     __ lea(rscratch2, ExternalAddress((address)&counter));
  85     __ ldrw(rscratch1, Address(rscratch2));
  86     __ addw(rscratch1, rscratch1, 1);
  87     __ strw(rscratch1, Address(rscratch2));
  88   }
  89 #define inc_counter_np(counter) \
  90   BLOCK_COMMENT("inc_counter " #counter); \
  91   inc_counter_np_(counter);
  92 #endif
  93 
  94   // Call stubs are used to call Java from C

 331 
 332     // pop parameters
 333     __ sub(esp, rfp, -sp_after_call_off * wordSize);
 334 
 335 #ifdef ASSERT
 336     // verify that threads correspond
 337     {
 338       Label L, S;
 339       __ ldr(rscratch1, thread);
 340       __ cmp(rthread, rscratch1);
 341       __ br(Assembler::NE, S);
 342       __ get_thread(rscratch1);
 343       __ cmp(rthread, rscratch1);
 344       __ br(Assembler::EQ, L);
 345       __ BIND(S);
 346       __ stop("StubRoutines::call_stub: threads must correspond");
 347       __ BIND(L);
 348     }
 349 #endif
 350 


 351     // restore callee-save registers
 352     __ ldpd(v15, v14,  d15_save);
 353     __ ldpd(v13, v12,  d13_save);
 354     __ ldpd(v11, v10,  d11_save);
 355     __ ldpd(v9,  v8,   d9_save);
 356 
 357     __ ldp(r28, r27,   r28_save);
 358     __ ldp(r26, r25,   r26_save);
 359     __ ldp(r24, r23,   r24_save);
 360     __ ldp(r22, r21,   r22_save);
 361     __ ldp(r20, r19,   r20_save);
 362 
 363     __ ldp(c_rarg0, c_rarg1,  call_wrapper);
 364     __ ldrw(c_rarg2, result_type);
 365     __ ldr(c_rarg3,  method);
 366     __ ldp(c_rarg4, c_rarg5,  entry_point);
 367     __ ldp(c_rarg6, c_rarg7,  parameter_size);
 368 
 369     // leave frame and return to caller
 370     __ leave();

6288     gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
6289 
6290     AtomicStubMark mark_cmpxchg_4_release
6291       (_masm, &aarch64_atomic_cmpxchg_4_release_impl);
6292     gen_cas_entry(MacroAssembler::word, memory_order_release);
6293     AtomicStubMark mark_cmpxchg_8_release
6294       (_masm, &aarch64_atomic_cmpxchg_8_release_impl);
6295     gen_cas_entry(MacroAssembler::xword, memory_order_release);
6296 
6297     AtomicStubMark mark_cmpxchg_4_seq_cst
6298       (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
6299     gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
6300     AtomicStubMark mark_cmpxchg_8_seq_cst
6301       (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
6302     gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
6303 
6304     ICache::invalidate_range(first_entry, __ pc() - first_entry);
6305   }
6306 #endif // LINUX
6307 









































































































































































































































































































6308   // Continuation point for throwing of implicit exceptions that are
6309   // not handled in the current activation. Fabricates an exception
6310   // oop and initiates normal exception dispatching in this
6311   // frame. Since we need to preserve callee-saved values (currently
6312   // only for C2, but done for C1 as well) we need a callee-saved oop
6313   // map and therefore have to make these stubs into RuntimeStubs
6314   // rather than BufferBlobs.  If the compiler needs all registers to
6315   // be preserved between the fault point and the exception handler
6316   // then it must assume responsibility for that in
6317   // AbstractCompiler::continuation_for_implicit_null_exception or
6318   // continuation_for_implicit_division_by_zero_exception. All other
6319   // implicit exceptions (e.g., NullPointerException or
6320   // AbstractMethodError on entry) are either at call sites or
6321   // otherwise assume that stack unwinding will be initiated, so
6322   // caller saved registers were assumed volatile in the compiler.
6323 
6324 #undef __
6325 #define __ masm->
6326 
6327   address generate_throw_exception(const char* name,

7280     //   StubRoutines::_dlog = generate_dlog();
7281     // }
7282 
7283     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
7284       StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
7285     }
7286 
7287     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
7288       StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
7289     }
7290 
7291     // Safefetch stubs.
7292     generate_safefetch("SafeFetch32", sizeof(int),     &StubRoutines::_safefetch32_entry,
7293                                                        &StubRoutines::_safefetch32_fault_pc,
7294                                                        &StubRoutines::_safefetch32_continuation_pc);
7295     generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
7296                                                        &StubRoutines::_safefetchN_fault_pc,
7297                                                        &StubRoutines::_safefetchN_continuation_pc);
7298   }
7299 














7300   void generate_all() {
7301     // support for verify_oop (must happen after universe_init)
7302     StubRoutines::_verify_oop_subroutine_entry     = generate_verify_oop();
7303     StubRoutines::_throw_AbstractMethodError_entry =
7304       generate_throw_exception("AbstractMethodError throw_exception",
7305                                CAST_FROM_FN_PTR(address,
7306                                                 SharedRuntime::
7307                                                 throw_AbstractMethodError));
7308 
7309     StubRoutines::_throw_IncompatibleClassChangeError_entry =
7310       generate_throw_exception("IncompatibleClassChangeError throw_exception",
7311                                CAST_FROM_FN_PTR(address,
7312                                                 SharedRuntime::
7313                                                 throw_IncompatibleClassChangeError));
7314 
7315     StubRoutines::_throw_NullPointerException_at_call_entry =
7316       generate_throw_exception("NullPointerException at call throw_exception",
7317                                CAST_FROM_FN_PTR(address,
7318                                                 SharedRuntime::
7319                                                 throw_NullPointerException_at_call));

7414     if (UseSHA3Intrinsics) {
7415       StubRoutines::_sha3_implCompress     = generate_sha3_implCompress(false,   "sha3_implCompress");
7416       StubRoutines::_sha3_implCompressMB   = generate_sha3_implCompress(true,    "sha3_implCompressMB");
7417     }
7418 
7419     // generate Adler32 intrinsics code
7420     if (UseAdler32Intrinsics) {
7421       StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32();
7422     }
7423 
7424 #ifdef LINUX
7425 
7426     generate_atomic_entry_points();
7427 
7428 #endif // LINUX
7429 
7430     StubRoutines::aarch64::set_completed();
7431   }
7432 
7433  public:
7434   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
7435     if (all) {
7436       generate_all();
7437     } else {
7438       generate_initial();




7439     }
7440   }
7441 }; // end class declaration
7442 
7443 #define UCM_TABLE_MAX_ENTRIES 8
7444 void StubGenerator_generate(CodeBuffer* code, bool all) {
7445   if (UnsafeCopyMemory::_table == NULL) {
7446     UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES);
7447   }
7448   StubGenerator g(code, all);
7449 }
7450 
7451 
7452 #ifdef LINUX
7453 
7454 // Define pointers to atomic stubs and initialize them to point to the
7455 // code in atomic_aarch64.S.
7456 
7457 #define DEFAULT_ATOMIC_OP(OPNAME, SIZE, RELAXED)                                \
7458   extern "C" uint64_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl \
7459     (volatile void *ptr, uint64_t arg1, uint64_t arg2);                 \
7460   aarch64_atomic_stub_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _impl \
7461     = aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl;
7462 
7463 DEFAULT_ATOMIC_OP(fetch_add, 4, )
7464 DEFAULT_ATOMIC_OP(fetch_add, 8, )
7465 DEFAULT_ATOMIC_OP(fetch_add, 4, _relaxed)
7466 DEFAULT_ATOMIC_OP(fetch_add, 8, _relaxed)
7467 DEFAULT_ATOMIC_OP(xchg, 4, )
7468 DEFAULT_ATOMIC_OP(xchg, 8, )
7469 DEFAULT_ATOMIC_OP(cmpxchg, 1, )
7470 DEFAULT_ATOMIC_OP(cmpxchg, 4, )
7471 DEFAULT_ATOMIC_OP(cmpxchg, 8, )
7472 DEFAULT_ATOMIC_OP(cmpxchg, 1, _relaxed)
7473 DEFAULT_ATOMIC_OP(cmpxchg, 4, _relaxed)
7474 DEFAULT_ATOMIC_OP(cmpxchg, 8, _relaxed)
7475 DEFAULT_ATOMIC_OP(cmpxchg, 4, _release)
7476 DEFAULT_ATOMIC_OP(cmpxchg, 8, _release)
7477 DEFAULT_ATOMIC_OP(cmpxchg, 4, _seq_cst)
7478 DEFAULT_ATOMIC_OP(cmpxchg, 8, _seq_cst)
7479 
7480 #undef DEFAULT_ATOMIC_OP
7481 
7482 #endif // LINUX






































































  31 #include "gc/shared/barrierSet.hpp"
  32 #include "gc/shared/barrierSetAssembler.hpp"
  33 #include "gc/shared/gc_globals.hpp"
  34 #include "gc/shared/tlab_globals.hpp"
  35 #include "interpreter/interpreter.hpp"
  36 #include "memory/universe.hpp"
  37 #include "nativeInst_aarch64.hpp"
  38 #include "oops/instanceOop.hpp"
  39 #include "oops/method.hpp"
  40 #include "oops/objArrayKlass.hpp"
  41 #include "oops/oop.inline.hpp"
  42 #include "prims/methodHandles.hpp"
  43 #include "runtime/atomic.hpp"
  44 #include "runtime/frame.inline.hpp"
  45 #include "runtime/handles.inline.hpp"
  46 #include "runtime/sharedRuntime.hpp"
  47 #include "runtime/stubCodeGenerator.hpp"
  48 #include "runtime/stubRoutines.hpp"
  49 #include "runtime/thread.inline.hpp"
  50 #include "utilities/align.hpp"
  51 #include "utilities/globalDefinitions.hpp"
  52 #include "utilities/powerOfTwo.hpp"
  53 #ifdef COMPILER2
  54 #include "opto/runtime.hpp"
  55 #endif
  56 #if INCLUDE_ZGC
  57 #include "gc/z/zThreadLocalData.hpp"
  58 #endif
  59 
  60 // Declaration and definition of StubGenerator (no .hpp file).
  61 // For a more detailed description of the stub routine structure
  62 // see the comment in stubRoutines.hpp
  63 
  64 #undef __
  65 #define __ _masm->
  66 #define TIMES_OOP Address::sxtw(exact_log2(UseCompressedOops ? 4 : 8))
  67 
  68 #ifdef PRODUCT
  69 #define BLOCK_COMMENT(str) /* nothing */
  70 #else
  71 #define BLOCK_COMMENT(str) __ block_comment(str)
  72 #endif
  73 
  74 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  75 
  76 OopMap* continuation_enter_setup(MacroAssembler* masm, int& stack_slots);
  77 void fill_continuation_entry(MacroAssembler* masm);
  78 void continuation_enter_cleanup(MacroAssembler* masm);
  79 
  80 // Stub Code definitions
  81 
  82 class StubGenerator: public StubCodeGenerator {
  83  private:
  84 
  85 #ifdef PRODUCT
  86 #define inc_counter_np(counter) ((void)0)
  87 #else
  88   void inc_counter_np_(int& counter) {
  89     __ lea(rscratch2, ExternalAddress((address)&counter));
  90     __ ldrw(rscratch1, Address(rscratch2));
  91     __ addw(rscratch1, rscratch1, 1);
  92     __ strw(rscratch1, Address(rscratch2));
  93   }
  94 #define inc_counter_np(counter) \
  95   BLOCK_COMMENT("inc_counter " #counter); \
  96   inc_counter_np_(counter);
  97 #endif
  98 
  99   // Call stubs are used to call Java from C

 336 
 337     // pop parameters
 338     __ sub(esp, rfp, -sp_after_call_off * wordSize);
 339 
 340 #ifdef ASSERT
 341     // verify that threads correspond
 342     {
 343       Label L, S;
 344       __ ldr(rscratch1, thread);
 345       __ cmp(rthread, rscratch1);
 346       __ br(Assembler::NE, S);
 347       __ get_thread(rscratch1);
 348       __ cmp(rthread, rscratch1);
 349       __ br(Assembler::EQ, L);
 350       __ BIND(S);
 351       __ stop("StubRoutines::call_stub: threads must correspond");
 352       __ BIND(L);
 353     }
 354 #endif
 355 
 356     __ pop_cont_fastpath(rthread);
 357 
 358     // restore callee-save registers
 359     __ ldpd(v15, v14,  d15_save);
 360     __ ldpd(v13, v12,  d13_save);
 361     __ ldpd(v11, v10,  d11_save);
 362     __ ldpd(v9,  v8,   d9_save);
 363 
 364     __ ldp(r28, r27,   r28_save);
 365     __ ldp(r26, r25,   r26_save);
 366     __ ldp(r24, r23,   r24_save);
 367     __ ldp(r22, r21,   r22_save);
 368     __ ldp(r20, r19,   r20_save);
 369 
 370     __ ldp(c_rarg0, c_rarg1,  call_wrapper);
 371     __ ldrw(c_rarg2, result_type);
 372     __ ldr(c_rarg3,  method);
 373     __ ldp(c_rarg4, c_rarg5,  entry_point);
 374     __ ldp(c_rarg6, c_rarg7,  parameter_size);
 375 
 376     // leave frame and return to caller
 377     __ leave();

6295     gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
6296 
6297     AtomicStubMark mark_cmpxchg_4_release
6298       (_masm, &aarch64_atomic_cmpxchg_4_release_impl);
6299     gen_cas_entry(MacroAssembler::word, memory_order_release);
6300     AtomicStubMark mark_cmpxchg_8_release
6301       (_masm, &aarch64_atomic_cmpxchg_8_release_impl);
6302     gen_cas_entry(MacroAssembler::xword, memory_order_release);
6303 
6304     AtomicStubMark mark_cmpxchg_4_seq_cst
6305       (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
6306     gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
6307     AtomicStubMark mark_cmpxchg_8_seq_cst
6308       (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
6309     gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
6310 
6311     ICache::invalidate_range(first_entry, __ pc() - first_entry);
6312   }
6313 #endif // LINUX
6314 
6315 RuntimeStub* generate_cont_doYield() {
6316     const char *name = "cont_doYield";
6317 
6318     enum layout {
6319       rfp_off1,
6320       rfp_off2,
6321       lr_off,
6322       lr_off2,
6323       framesize // inclusive of return address
6324     };
6325     // assert(is_even(framesize/2), "sp not 16-byte aligned");
6326     
6327     int insts_size = 512;
6328     int locs_size  = 64;
6329     CodeBuffer code(name, insts_size, locs_size);
6330     OopMapSet* oop_maps  = new OopMapSet();
6331     MacroAssembler* masm = new MacroAssembler(&code);
6332     MacroAssembler* _masm = masm;
6333 
6334     address start = __ pc();
6335 
6336     __ enter();
6337 
6338     __ mov(c_rarg1, sp);
6339 
6340     int frame_complete = __ pc() - start;
6341     address the_pc = __ pc();
6342 
6343     __ post_call_nop(); // this must be exactly after the pc value that is pushed into the frame info, we use this nop for fast CodeBlob lookup
6344 
6345     __ mov(c_rarg0, rthread);
6346     __ set_last_Java_frame(sp, rfp, the_pc, rscratch1);
6347 
6348     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::freeze), 2);
6349       
6350     __ reset_last_Java_frame(true);
6351 
6352     Label pinned;
6353 
6354     __ cbnz(r0, pinned);
6355 
6356     __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
6357     __ mov(sp, rscratch1);
6358     continuation_enter_cleanup(masm);
6359 
6360     __ bind(pinned); // pinned -- return to caller
6361     
6362     __ leave();
6363     __ ret(lr);
6364 
6365     OopMap* map = new OopMap(framesize, 1);
6366     // map->set_callee_saved(VMRegImpl::stack2reg(rfp_off), rfp->as_VMReg());
6367     oop_maps->add_gc_map(the_pc - start, map);
6368 
6369     RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size)
6370     RuntimeStub::new_runtime_stub(name,
6371                                   &code,
6372                                   frame_complete,
6373                                   (framesize >> (LogBytesPerWord - LogBytesPerInt)),
6374                                   oop_maps, false);
6375     return stub;
6376   }
6377 
6378   address generate_cont_jump_from_safepoint() {
6379     __ align(CodeEntryAlignment);
6380     StubCodeMark mark(this, "StubRoutines","Continuation jump from safepoint");
6381 
6382     address start = __ pc();
6383 
6384 #ifdef ASSERT
6385     { // verify that threads correspond
6386       Label L;
6387       __ get_thread(rscratch1);
6388       __ cmp(rthread, rscratch1);
6389       __ br(Assembler::EQ, L);
6390       __ stop("StubRoutines::cont_jump_from_safepoint: threads must correspond");
6391       __ BIND(L);
6392     }
6393 #endif
6394 
6395     __ reset_last_Java_frame(true); // false would be fine, too, I guess
6396     __ reinit_heapbase();
6397     
6398     __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
6399     __ mov(sp, rscratch1);
6400     continuation_enter_cleanup(_masm);
6401     __ leave();
6402     __ ret(lr);
6403 
6404     return start;
6405   }
6406 
6407   address generate_cont_thaw(bool return_barrier, bool exception) {
6408     assert (return_barrier || !exception, "must be");
6409 
6410     address start = __ pc();
6411 
6412     if (return_barrier) {
6413       __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
6414       __ mov(sp, rscratch1);
6415     }
6416     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
6417 
6418     if (return_barrier) {
6419       // preserve possible return value from a method returning to the return barrier
6420       __ fmovd(rscratch1, v0);
6421       __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
6422     }
6423 
6424     __ movw(c_rarg1, (return_barrier ? 1 : 0) + (exception ? 1 : 0));
6425     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
6426     __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
6427 
6428     if (return_barrier) {
6429       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
6430       __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
6431       __ fmovd(v0, rscratch1);
6432     }
6433     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
6434 
6435 
6436     Label thaw_success;
6437     // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
6438     __ cbnz(rscratch2, thaw_success);
6439     __ lea(rscratch1, ExternalAddress(StubRoutines::throw_StackOverflowError_entry()));
6440     __ br(rscratch1);
6441     __ bind(thaw_success);
6442     
6443     // make room for the thawed frames
6444     __ sub(rscratch1, sp, rscratch2);
6445     __ andr(rscratch1, rscratch1, -16); // align
6446     __ mov(sp, rscratch1);
6447     
6448     if (return_barrier) {
6449       // save original return value -- again
6450       __ fmovd(rscratch1, v0);
6451       __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
6452     }
6453 
6454     __ movw(c_rarg1, (return_barrier ? 1 : 0) + (exception ? 1 : 0));
6455     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::thaw), rthread, c_rarg1);
6456     __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
6457 
6458     if (return_barrier) {
6459       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
6460       __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
6461       __ fmovd(v0, rscratch1);
6462     } else {
6463       __ mov(r0, zr); // return 0 (success) from doYield
6464     }
6465 
6466     // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
6467     __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
6468     __ mov(rfp, sp);
6469 
6470     if (exception) {
6471       __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
6472       __ verify_oop(r0);
6473       __ mov(r19, r0); // save return value contaning the exception oop in callee-saved R19
6474 
6475       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
6476 
6477       // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
6478       // __ reinitialize_ptrue();
6479 
6480       // see OptoRuntime::generate_exception_blob: r0 -- exception oop, r3 -- exception pc
6481 
6482       __ mov(r1, r0); // the exception handler
6483       __ mov(r0, r19); // restore return value contaning the exception oop
6484       __ verify_oop(r0);
6485 
6486       __ leave();
6487       __ mov(r3, lr);
6488       __ br(r1); // the exception handler
6489     }
6490 
6491     // We're "returning" into the topmost thawed frame; see Thaw::push_return_frame
6492     __ leave();
6493     __ ret(lr);
6494 
6495     return start;
6496   }
6497 
6498   address generate_cont_thaw() {
6499     StubCodeMark mark(this, "StubRoutines", "Cont thaw");
6500     address start = __ pc();
6501     generate_cont_thaw(false, false);
6502     return start;
6503   }
6504 
6505   address generate_cont_returnBarrier() {
6506     // TODO: will probably need multiple return barriers depending on return type
6507     StubCodeMark mark(this, "StubRoutines", "cont return barrier");
6508     address start = __ pc();
6509 
6510     generate_cont_thaw(true, false);
6511 
6512     return start;
6513   }
6514 
6515   address generate_cont_returnBarrier_exception() {
6516     StubCodeMark mark(this, "StubRoutines", "cont return barrier exception handler");
6517     address start = __ pc();
6518 
6519     generate_cont_thaw(true, true);
6520 
6521     return start;
6522   }
6523 
6524   address generate_cont_interpreter_forced_preempt_return() {
6525       StubCodeMark mark(this, "StubRoutines", "cont interpreter forced preempt return");
6526       address start = __ pc();
6527 
6528       // This is necessary for forced yields, as the return addres (in rbx) is captured in a call_VM, and skips the restoration of rbcp and locals
6529 
6530       assert_asm(_masm, __ cmp(sp, rfp), Assembler::EQ, "sp != fp"); // __ mov(rfp, sp);
6531       __ leave(); // we're now on the last thawed frame
6532 
6533       __ ldr(rbcp,    Address(rfp, frame::interpreter_frame_bcp_offset    * wordSize)); // InterpreterMacroAssembler::restore_bcp()
6534       __ ldr(rlocals, Address(rfp, frame::interpreter_frame_locals_offset * wordSize)); // InterpreterMacroAssembler::restore_locals()
6535       __ ldr(rcpool,  Address(rfp, frame::interpreter_frame_cache_offset  * wordSize)); // InterpreterMacroAssembler::restore_constant_pool_cache()
6536       __ ldr(rmethod, Address(rfp, frame::interpreter_frame_method_offset * wordSize)); // InterpreterMacroAssembler::get_method(rmethod) -- might not be necessary
6537       // __ reinit_heapbase();
6538 
6539       // Restore stack bottom in case i2c adjusted stack and NULL it as marker that esp is now tos until next java call
6540       __ ldr(esp, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
6541       __ str(zr,  Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
6542 
6543       __ ret(lr);
6544 
6545       return start;
6546     }
6547 
6548 #if INCLUDE_JFR
6549 
6550   static void jfr_set_last_java_frame(MacroAssembler* _masm, Register thread) {
6551     Register last_java_pc = c_rarg0;
6552     Register last_java_sp = c_rarg2;
6553     __ ldr(last_java_pc, Address(sp, 0));
6554     __ lea(last_java_sp, Address(sp, wordSize));
6555     // __ vzeroupper();
6556     Address anchor_java_pc(thread, JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
6557     __ str(last_java_pc, anchor_java_pc);
6558     __ str(last_java_sp, Address(thread, JavaThread::last_Java_sp_offset()));
6559   }
6560 
6561   static void jfr_prologue(MacroAssembler* _masm, Register thread) {
6562     jfr_set_last_java_frame(_masm, thread);
6563     __ mov(c_rarg0, rthread);
6564   }
6565 
6566   // Handle is dereference here using correct load constructs.
6567   static void jfr_epilogue(MacroAssembler* _masm, Register thread) {
6568     __ reset_last_Java_frame(false);
6569     Label null_jobject;
6570     __ cbz(r0, null_jobject);
6571     DecoratorSet decorators = ACCESS_READ | IN_NATIVE;
6572     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
6573     bs->load_at(_masm, decorators, T_OBJECT, r0, Address(r0, 0), rscratch1, rthread);
6574     __ bind(null_jobject);
6575   }
6576 
6577   // For c2: c_rarg0 is junk, c_rarg1 is the thread id. Call to runtime to write a checkpoint.
6578   // Runtime will return a jobject handle to the event writer. The handle is dereferenced and the return value
6579   // is the event writer oop.
6580   address generate_jfr_write_checkpoint() {
6581     StubCodeMark mark(this, "jfr_write_checkpoint", "JFR C2 support for Virtual Threads");
6582     address start = __ pc();
6583 
6584     __ enter();
6585     jfr_prologue(_masm, rthread);
6586     __ call_VM_leaf(CAST_FROM_FN_PTR(address, JFR_WRITE_CHECKPOINT_FUNCTION), 2);
6587     jfr_epilogue(_masm, rthread);
6588     __ leave();
6589     __ ret(lr);
6590 
6591     return start;
6592   }
6593 
6594   // For c1: call the corresponding runtime routine, it returns a jobject handle to the event writer.
6595   // The handle is dereferenced and the return value is the event writer oop.
6596   address generate_jfr_get_event_writer() {
6597     StubCodeMark mark(this, "jfr_get_event_writer", "JFR C1 support for Virtual Threads");
6598     address start = __ pc();
6599 
6600     __ enter();
6601     jfr_prologue(_masm, rthread);
6602     __ call_VM_leaf(CAST_FROM_FN_PTR(address, JFR_GET_EVENT_WRITER_FUNCTION), 1);
6603     jfr_epilogue(_masm, rthread);
6604     __ leave();
6605     __ ret(lr);
6606 
6607     return start;
6608   }
6609 
6610 #endif // INCLUDE_JFR
6611 
6612   // Continuation point for throwing of implicit exceptions that are
6613   // not handled in the current activation. Fabricates an exception
6614   // oop and initiates normal exception dispatching in this
6615   // frame. Since we need to preserve callee-saved values (currently
6616   // only for C2, but done for C1 as well) we need a callee-saved oop
6617   // map and therefore have to make these stubs into RuntimeStubs
6618   // rather than BufferBlobs.  If the compiler needs all registers to
6619   // be preserved between the fault point and the exception handler
6620   // then it must assume responsibility for that in
6621   // AbstractCompiler::continuation_for_implicit_null_exception or
6622   // continuation_for_implicit_division_by_zero_exception. All other
6623   // implicit exceptions (e.g., NullPointerException or
6624   // AbstractMethodError on entry) are either at call sites or
6625   // otherwise assume that stack unwinding will be initiated, so
6626   // caller saved registers were assumed volatile in the compiler.
6627 
6628 #undef __
6629 #define __ masm->
6630 
6631   address generate_throw_exception(const char* name,

7584     //   StubRoutines::_dlog = generate_dlog();
7585     // }
7586 
7587     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
7588       StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
7589     }
7590 
7591     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
7592       StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
7593     }
7594 
7595     // Safefetch stubs.
7596     generate_safefetch("SafeFetch32", sizeof(int),     &StubRoutines::_safefetch32_entry,
7597                                                        &StubRoutines::_safefetch32_fault_pc,
7598                                                        &StubRoutines::_safefetch32_continuation_pc);
7599     generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
7600                                                        &StubRoutines::_safefetchN_fault_pc,
7601                                                        &StubRoutines::_safefetchN_continuation_pc);
7602   }
7603 
7604   void generate_phase1() {
7605     // Continuation stubs:
7606     StubRoutines::_cont_thaw          = generate_cont_thaw();
7607     StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
7608     StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
7609     StubRoutines::_cont_doYield_stub = generate_cont_doYield();
7610     StubRoutines::_cont_doYield    = StubRoutines::_cont_doYield_stub->entry_point();
7611     StubRoutines::_cont_jump_from_sp = generate_cont_jump_from_safepoint();
7612     StubRoutines::_cont_interpreter_forced_preempt_return = generate_cont_interpreter_forced_preempt_return();
7613 
7614     JFR_ONLY(StubRoutines::_jfr_write_checkpoint = generate_jfr_write_checkpoint();)
7615     JFR_ONLY(StubRoutines::_jfr_get_event_writer = generate_jfr_get_event_writer();)
7616   }
7617 
7618   void generate_all() {
7619     // support for verify_oop (must happen after universe_init)
7620     StubRoutines::_verify_oop_subroutine_entry     = generate_verify_oop();
7621     StubRoutines::_throw_AbstractMethodError_entry =
7622       generate_throw_exception("AbstractMethodError throw_exception",
7623                                CAST_FROM_FN_PTR(address,
7624                                                 SharedRuntime::
7625                                                 throw_AbstractMethodError));
7626 
7627     StubRoutines::_throw_IncompatibleClassChangeError_entry =
7628       generate_throw_exception("IncompatibleClassChangeError throw_exception",
7629                                CAST_FROM_FN_PTR(address,
7630                                                 SharedRuntime::
7631                                                 throw_IncompatibleClassChangeError));
7632 
7633     StubRoutines::_throw_NullPointerException_at_call_entry =
7634       generate_throw_exception("NullPointerException at call throw_exception",
7635                                CAST_FROM_FN_PTR(address,
7636                                                 SharedRuntime::
7637                                                 throw_NullPointerException_at_call));

7732     if (UseSHA3Intrinsics) {
7733       StubRoutines::_sha3_implCompress     = generate_sha3_implCompress(false,   "sha3_implCompress");
7734       StubRoutines::_sha3_implCompressMB   = generate_sha3_implCompress(true,    "sha3_implCompressMB");
7735     }
7736 
7737     // generate Adler32 intrinsics code
7738     if (UseAdler32Intrinsics) {
7739       StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32();
7740     }
7741 
7742 #ifdef LINUX
7743 
7744     generate_atomic_entry_points();
7745 
7746 #endif // LINUX
7747 
7748     StubRoutines::aarch64::set_completed();
7749   }
7750 
7751  public:
7752   StubGenerator(CodeBuffer* code, int phase) : StubCodeGenerator(code) {
7753     if (phase == 0) {


7754       generate_initial();
7755     } else if (phase == 1) {
7756       generate_phase1(); // stubs that must be available for the interpreter
7757     } else {
7758       generate_all();
7759     }
7760   }
7761 }; // end class declaration
7762 
7763 #define UCM_TABLE_MAX_ENTRIES 8
7764 void StubGenerator_generate(CodeBuffer* code, int phase) {
7765   if (UnsafeCopyMemory::_table == NULL) {
7766     UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES);
7767   }
7768   StubGenerator g(code, phase);
7769 }
7770 
7771 
7772 #ifdef LINUX
7773 
7774 // Define pointers to atomic stubs and initialize them to point to the
7775 // code in atomic_aarch64.S.
7776 
7777 #define DEFAULT_ATOMIC_OP(OPNAME, SIZE, RELAXED)                                \
7778   extern "C" uint64_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl \
7779     (volatile void *ptr, uint64_t arg1, uint64_t arg2);                 \
7780   aarch64_atomic_stub_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _impl \
7781     = aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl;
7782 
7783 DEFAULT_ATOMIC_OP(fetch_add, 4, )
7784 DEFAULT_ATOMIC_OP(fetch_add, 8, )
7785 DEFAULT_ATOMIC_OP(fetch_add, 4, _relaxed)
7786 DEFAULT_ATOMIC_OP(fetch_add, 8, _relaxed)
7787 DEFAULT_ATOMIC_OP(xchg, 4, )
7788 DEFAULT_ATOMIC_OP(xchg, 8, )
7789 DEFAULT_ATOMIC_OP(cmpxchg, 1, )
7790 DEFAULT_ATOMIC_OP(cmpxchg, 4, )
7791 DEFAULT_ATOMIC_OP(cmpxchg, 8, )
7792 DEFAULT_ATOMIC_OP(cmpxchg, 1, _relaxed)
7793 DEFAULT_ATOMIC_OP(cmpxchg, 4, _relaxed)
7794 DEFAULT_ATOMIC_OP(cmpxchg, 8, _relaxed)
7795 DEFAULT_ATOMIC_OP(cmpxchg, 4, _release)
7796 DEFAULT_ATOMIC_OP(cmpxchg, 8, _release)
7797 DEFAULT_ATOMIC_OP(cmpxchg, 4, _seq_cst)
7798 DEFAULT_ATOMIC_OP(cmpxchg, 8, _seq_cst)
7799 
7800 #undef DEFAULT_ATOMIC_OP
7801 
7802 #endif // LINUX
7803 
7804 
7805 #undef __
7806 #define __ masm->
7807 
7808 // on exit, sp points to the ContinuationEntry
7809 OopMap* continuation_enter_setup(MacroAssembler* masm, int& stack_slots) {
7810   assert (ContinuationEntry::size() % VMRegImpl::stack_slot_size == 0, "");
7811   assert (in_bytes(ContinuationEntry::cont_offset())  % VMRegImpl::stack_slot_size == 0, "");
7812   assert (in_bytes(ContinuationEntry::chunk_offset()) % VMRegImpl::stack_slot_size == 0, "");
7813 
7814   stack_slots += (int)ContinuationEntry::size()/wordSize;
7815   __ sub(sp, sp, (int)ContinuationEntry::size()); // place Continuation metadata
7816 
7817   OopMap* map = new OopMap(((int)ContinuationEntry::size() + wordSize)/ VMRegImpl::stack_slot_size, 0 /* arg_slots*/);
7818   ContinuationEntry::setup_oopmap(map);
7819 
7820   __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
7821   __ str(rscratch1, Address(sp, ContinuationEntry::parent_offset()));
7822   __ mov(rscratch1, sp); // we can't use sp as the source in str
7823   __ str(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
7824 
7825   return map;
7826 }
7827 
7828 // on entry c_rarg1 points to the continuation 
7829 //          sp points to ContinuationEntry
7830 void fill_continuation_entry(MacroAssembler* masm) {
7831 #ifdef ASSERT
7832   __ movw(rscratch1, 0x1234);
7833   __ strw(rscratch1, Address(sp, ContinuationEntry::cookie_offset()));
7834 #endif
7835 
7836   __ str(c_rarg1, Address(sp, ContinuationEntry::cont_offset()));
7837   __ str(zr, Address(sp, ContinuationEntry::chunk_offset()));
7838   __ strw(zr, Address(sp, ContinuationEntry::argsize_offset()));
7839 
7840   __ ldr(rscratch1, Address(rthread, JavaThread::cont_fastpath_offset()));
7841   __ str(rscratch1, Address(sp, ContinuationEntry::parent_cont_fastpath_offset()));
7842   __ ldr(rscratch1, Address(rthread, JavaThread::held_monitor_count_offset()));
7843   __ str(rscratch1, Address(sp, ContinuationEntry::parent_held_monitor_count_offset()));
7844   
7845   __ str(zr, Address(rthread, JavaThread::cont_fastpath_offset()));
7846   __ reset_held_monitor_count(rthread);
7847 }
7848 
7849 // on entry, sp points to the ContinuationEntry
7850 // on exit, rfp points to the spilled rfp in the entry frame
7851 void continuation_enter_cleanup(MacroAssembler* masm) {
7852 #ifndef PRODUCT
7853   Label OK;
7854   __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
7855   __ cmp(sp, rscratch1);
7856   __ br(Assembler::EQ, OK);
7857   __ stop("incorrect sp1");
7858   __ bind(OK);
7859 #endif
7860   
7861   __ ldr(rscratch1, Address(sp, ContinuationEntry::parent_cont_fastpath_offset()));
7862   __ str(rscratch1, Address(rthread, JavaThread::cont_fastpath_offset()));
7863   __ ldr(rscratch1, Address(sp, ContinuationEntry::parent_held_monitor_count_offset()));
7864   __ str(rscratch1, Address(rthread, JavaThread::held_monitor_count_offset()));
7865 
7866   __ ldr(rscratch2, Address(sp, ContinuationEntry::parent_offset()));
7867   __ str(rscratch2, Address(rthread, JavaThread::cont_entry_offset()));
7868   __ add(rfp, sp, (int)ContinuationEntry::size());
7869 }
7870 
7871 #undef __
< prev index next >