165 flags_off, flagsH_off,
166 // The frame sender code expects that rbp will be in the "natural" place and
167 // will override any oopMap setting for it. We must therefore force the layout
168 // so that it agrees with the frame sender code.
169 rbp_off, rbpH_off, // copy of rbp we will restore
170 return_off, returnH_off, // slot for return address
171 reg_save_size // size in compiler stack slots
172 };
173
174 public:
175 static OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_wide_vectors);
176 static void restore_live_registers(MacroAssembler* masm, bool restore_wide_vectors = false);
177
178 // Offsets into the register save area
179 // Used by deoptimization when it is managing result register
180 // values on its own
181
182 static int rax_offset_in_bytes(void) { return BytesPerInt * rax_off; }
183 static int rdx_offset_in_bytes(void) { return BytesPerInt * rdx_off; }
184 static int rbx_offset_in_bytes(void) { return BytesPerInt * rbx_off; }
185 static int xmm0_offset_in_bytes(void) { return BytesPerInt * xmm0_off; }
186 static int return_offset_in_bytes(void) { return BytesPerInt * return_off; }
187
188 // During deoptimization only the result registers need to be restored,
189 // all the other values have already been extracted.
190 static void restore_result_registers(MacroAssembler* masm);
191 };
192
193 OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_wide_vectors) {
194 int off = 0;
195 int num_xmm_regs = XMMRegister::available_xmm_registers();
196 #if COMPILER2_OR_JVMCI
197 if (save_wide_vectors && UseAVX == 0) {
198 save_wide_vectors = false; // vectors larger than 16 byte long are supported only with AVX
199 }
200 assert(!save_wide_vectors || MaxVectorSize <= 64, "Only up to 64 byte long vectors are supported");
201 #else
202 save_wide_vectors = false; // vectors are generated only by C2 and JVMCI
203 #endif
204
1414 __ movptr(rax, Address(r15_thread, JavaThread::cont_fastpath_offset()));
1415 __ movptr(Address(rsp, ContinuationEntry::parent_cont_fastpath_offset()), rax);
1416 __ movq(rax, Address(r15_thread, JavaThread::held_monitor_count_offset()));
1417 __ movq(Address(rsp, ContinuationEntry::parent_held_monitor_count_offset()), rax);
1418
1419 __ movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), 0);
1420 __ movq(Address(r15_thread, JavaThread::held_monitor_count_offset()), 0);
1421 }
1422
1423 //---------------------------- continuation_enter_cleanup ---------------------------
1424 //
1425 // Arguments:
1426 // rsp: pointer to the ContinuationEntry
1427 //
1428 // Results:
1429 // rsp: pointer to the spilled rbp in the entry frame
1430 //
1431 // Kills:
1432 // rbx
1433 //
1434 void static continuation_enter_cleanup(MacroAssembler* masm) {
1435 #ifdef ASSERT
1436 Label L_good_sp;
1437 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
1438 __ jcc(Assembler::equal, L_good_sp);
1439 __ stop("Incorrect rsp at continuation_enter_cleanup");
1440 __ bind(L_good_sp);
1441 #endif
1442 __ movptr(rbx, Address(rsp, ContinuationEntry::parent_cont_fastpath_offset()));
1443 __ movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), rbx);
1444
1445 if (CheckJNICalls) {
1446 // Check if this is a virtual thread continuation
1447 Label L_skip_vthread_code;
1448 __ cmpl(Address(rsp, ContinuationEntry::flags_offset()), 0);
1449 __ jcc(Assembler::equal, L_skip_vthread_code);
1450
1451 // If the held monitor count is > 0 and this vthread is terminating then
1452 // it failed to release a JNI monitor. So we issue the same log message
1453 // that JavaThread::exit does.
1454 __ cmpptr(Address(r15_thread, JavaThread::jni_monitor_count_offset()), 0);
1604 address stub = CompiledDirectCall::emit_to_interp_stub(masm, __ pc());
1605 if (stub == nullptr) {
1606 fatal("CodeCache is full at gen_continuation_enter");
1607 }
1608
1609 // The call needs to be resolved. There's a special case for this in
1610 // SharedRuntime::find_callee_info_helper() which calls
1611 // LinkResolver::resolve_continuation_enter() which resolves the call to
1612 // Continuation.enter(Continuation c, boolean isContinue).
1613 __ call(resolve);
1614
1615 oop_maps->add_gc_map(__ pc() - start, map);
1616 __ post_call_nop();
1617
1618 __ jmpb(L_exit);
1619
1620 // --- Thawing path
1621
1622 __ bind(L_thaw);
1623
1624 __ call(RuntimeAddress(StubRoutines::cont_thaw()));
1625
1626 ContinuationEntry::_return_pc_offset = __ pc() - start;
1627 oop_maps->add_gc_map(__ pc() - start, map->deep_copy());
1628 __ post_call_nop();
1629
1630 // --- Normal exit (resolve/thawing)
1631
1632 __ bind(L_exit);
1633
1634 continuation_enter_cleanup(masm);
1635 __ pop(rbp);
1636 __ ret(0);
1637
1638 // --- Exception handling path
1639
1640 exception_offset = __ pc() - start;
1641
1642 continuation_enter_cleanup(masm);
1643 __ pop(rbp);
1706 continuation_enter_cleanup(masm);
1707 __ pop(rbp);
1708 __ ret(0);
1709
1710 __ bind(L_pinned);
1711
1712 // Pinned, return to caller
1713
1714 // handle pending exception thrown by freeze
1715 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD);
1716 Label ok;
1717 __ jcc(Assembler::equal, ok);
1718 __ leave();
1719 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
1720 __ bind(ok);
1721
1722 __ leave();
1723 __ ret(0);
1724 }
1725
1726 static void gen_special_dispatch(MacroAssembler* masm,
1727 const methodHandle& method,
1728 const BasicType* sig_bt,
1729 const VMRegPair* regs) {
1730 verify_oop_args(masm, method, sig_bt, regs);
1731 vmIntrinsics::ID iid = method->intrinsic_id();
1732
1733 // Now write the args into the outgoing interpreter space
1734 bool has_receiver = false;
1735 Register receiver_reg = noreg;
1736 int member_arg_pos = -1;
1737 Register member_reg = noreg;
1738 int ref_kind = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid);
1739 if (ref_kind != 0) {
1740 member_arg_pos = method->size_of_parameters() - 1; // trailing MemberName argument
1741 member_reg = rbx; // known to be free at this point
1742 has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind);
1743 } else if (iid == vmIntrinsics::_invokeBasic) {
1744 has_receiver = true;
1745 } else if (iid == vmIntrinsics::_linkToNative) {
2269
2270 // Test if the oopMark is an obvious stack pointer, i.e.,
2271 // 1) (mark & 3) == 0, and
2272 // 2) rsp <= mark < mark + os::pagesize()
2273 // These 3 tests can be done by evaluating the following
2274 // expression: ((mark - rsp) & (3 - os::vm_page_size())),
2275 // assuming both stack pointer and pagesize have their
2276 // least significant 2 bits clear.
2277 // NOTE: the oopMark is in swap_reg %rax as the result of cmpxchg
2278
2279 __ subptr(swap_reg, rsp);
2280 __ andptr(swap_reg, 3 - (int)os::vm_page_size());
2281
2282 // Save the test result, for recursive case, the result is zero
2283 __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2284 __ jcc(Assembler::notEqual, slow_path_lock);
2285 } else {
2286 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
2287 __ lightweight_lock(obj_reg, swap_reg, r15_thread, rscratch1, slow_path_lock);
2288 }
2289 __ bind(count_mon);
2290 __ inc_held_monitor_count();
2291
2292 // Slow path will re-enter here
2293 __ bind(lock_done);
2294 }
2295
2296 // Finally just about ready to make the JNI call
2297
2298 // get JNIEnv* which is first argument to native
2299 __ lea(c_rarg0, Address(r15_thread, in_bytes(JavaThread::jni_environment_offset())));
2300
2301 // Now set thread in native
2302 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native);
2303
2304 __ call(RuntimeAddress(native_func));
2305
2306 // Verify or restore cpu control state after JNI call
2307 __ restore_cpu_control_state_after_jni(rscratch1);
2308
2361 // by hand.
2362 //
2363 __ vzeroupper();
2364 save_native_result(masm, ret_type, stack_slots);
2365 __ mov(c_rarg0, r15_thread);
2366 __ mov(r12, rsp); // remember sp
2367 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
2368 __ andptr(rsp, -16); // align stack as required by ABI
2369 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
2370 __ mov(rsp, r12); // restore sp
2371 __ reinit_heapbase();
2372 // Restore any method result value
2373 restore_native_result(masm, ret_type, stack_slots);
2374 __ bind(Continue);
2375 }
2376
2377 // change thread state
2378 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java);
2379 __ bind(after_transition);
2380
2381 Label reguard;
2382 Label reguard_done;
2383 __ cmpl(Address(r15_thread, JavaThread::stack_guard_state_offset()), StackOverflow::stack_guard_yellow_reserved_disabled);
2384 __ jcc(Assembler::equal, reguard);
2385 __ bind(reguard_done);
2386
2387 // native result if any is live
2388
2389 // Unlock
2390 Label slow_path_unlock;
2391 Label unlock_done;
2392 if (method->is_synchronized()) {
2393
2394 Label fast_done;
2395
2396 // Get locked oop from the handle we passed to jni
2397 __ movptr(obj_reg, Address(oop_handle_reg, 0));
2398
2399 if (LockingMode == LM_LEGACY) {
2400 Label not_recur;
2401 // Simple recursive lock?
2402 __ cmpptr(Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size), NULL_WORD);
2403 __ jcc(Assembler::notEqual, not_recur);
2404 __ dec_held_monitor_count();
2405 __ jmpb(fast_done);
2406 __ bind(not_recur);
2407 }
2408
2409 // Must save rax if it is live now because cmpxchg must use it
2410 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2411 save_native_result(masm, ret_type, stack_slots);
2412 }
2413
2414 if (LockingMode == LM_MONITOR) {
2415 __ jmp(slow_path_unlock);
2416 } else if (LockingMode == LM_LEGACY) {
2417 // get address of the stack lock
2418 __ lea(rax, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2419 // get old displaced header
2420 __ movptr(old_hdr, Address(rax, 0));
2421
2422 // Atomic swap old header if oop still contains the stack lock
2423 __ lock();
2424 __ cmpxchgptr(old_hdr, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2425 __ jcc(Assembler::notEqual, slow_path_unlock);
2426 __ dec_held_monitor_count();
2427 } else {
2428 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
2429 __ lightweight_unlock(obj_reg, swap_reg, r15_thread, lock_reg, slow_path_unlock);
2430 __ dec_held_monitor_count();
2431 }
2432
2433 // slow path re-enters here
2434 __ bind(unlock_done);
2435 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2436 restore_native_result(masm, ret_type, stack_slots);
2437 }
2438
2439 __ bind(fast_done);
2440 }
2441 if (DTraceMethodProbes) {
2442 save_native_result(masm, ret_type, stack_slots);
2443 __ mov_metadata(c_rarg1, method());
2444 __ call_VM_leaf(
2445 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
2446 r15_thread, c_rarg1);
2447 restore_native_result(masm, ret_type, stack_slots);
2448 }
2449
2450 __ reset_last_Java_frame(false);
2485 // and forward the exception
2486 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2487
2488 // Slow path locking & unlocking
2489 if (method->is_synchronized()) {
2490
2491 // BEGIN Slow path lock
2492 __ bind(slow_path_lock);
2493
2494 // has last_Java_frame setup. No exceptions so do vanilla call not call_VM
2495 // args are (oop obj, BasicLock* lock, JavaThread* thread)
2496
2497 // protect the args we've loaded
2498 save_args(masm, total_c_args, c_arg, out_regs);
2499
2500 __ mov(c_rarg0, obj_reg);
2501 __ mov(c_rarg1, lock_reg);
2502 __ mov(c_rarg2, r15_thread);
2503
2504 // Not a leaf but we have last_Java_frame setup as we want
2505 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3);
2506 restore_args(masm, total_c_args, c_arg, out_regs);
2507
2508 #ifdef ASSERT
2509 { Label L;
2510 __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), NULL_WORD);
2511 __ jcc(Assembler::equal, L);
2512 __ stop("no pending exception allowed on exit from monitorenter");
2513 __ bind(L);
2514 }
2515 #endif
2516 __ jmp(lock_done);
2517
2518 // END Slow path lock
2519
2520 // BEGIN Slow path unlock
2521 __ bind(slow_path_unlock);
2522
2523 // If we haven't already saved the native result we must save it now as xmm registers
2524 // are still exposed.
2525 __ vzeroupper();
2577 __ mov(rsp, r12); // restore sp
2578 __ reinit_heapbase();
2579 restore_native_result(masm, ret_type, stack_slots);
2580 // and continue
2581 __ jmp(reguard_done);
2582
2583
2584
2585 __ flush();
2586
2587 nmethod *nm = nmethod::new_native_nmethod(method,
2588 compile_id,
2589 masm->code(),
2590 vep_offset,
2591 frame_complete,
2592 stack_slots / VMRegImpl::slots_per_word,
2593 (is_static ? in_ByteSize(klass_offset) : in_ByteSize(receiver_offset)),
2594 in_ByteSize(lock_slot_offset*VMRegImpl::stack_slot_size),
2595 oop_maps);
2596
2597 return nm;
2598 }
2599
2600 // this function returns the adjust size (in number of words) to a c2i adapter
2601 // activation for use during deoptimization
2602 int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals ) {
2603 return (callee_locals - callee_parameters) * Interpreter::stackElementWords;
2604 }
2605
2606
2607 uint SharedRuntime::out_preserve_stack_slots() {
2608 return 0;
2609 }
2610
2611
2612 // Number of stack slots between incoming argument block and the start of
2613 // a new frame. The PROLOG must add this many slots to the stack. The
2614 // EPILOG must remove this many slots. amd64 needs two slots for
2615 // return address.
2616 uint SharedRuntime::in_preserve_stack_slots() {
2617 return 4 + 2 * VerifyStackAtCalls;
2618 }
2619
2620 //------------------------------generate_deopt_blob----------------------------
2621 void SharedRuntime::generate_deopt_blob() {
2622 // Allocate space for the code
2623 ResourceMark rm;
2624 // Setup code generation tools
2625 int pad = 0;
2626 if (UseAVX > 2) {
2627 pad += 1024;
2628 }
2629 if (UseAPX) {
2630 pad += 1024;
2631 }
2632 #if INCLUDE_JVMCI
2633 if (EnableJVMCI) {
2634 pad += 512; // Increase the buffer size when compiling for JVMCI
2635 }
2636 #endif
2637 CodeBuffer buffer("deopt_blob", 2560+pad, 1024);
2638 MacroAssembler* masm = new MacroAssembler(&buffer);
2639 int frame_size_in_words;
|
165 flags_off, flagsH_off,
166 // The frame sender code expects that rbp will be in the "natural" place and
167 // will override any oopMap setting for it. We must therefore force the layout
168 // so that it agrees with the frame sender code.
169 rbp_off, rbpH_off, // copy of rbp we will restore
170 return_off, returnH_off, // slot for return address
171 reg_save_size // size in compiler stack slots
172 };
173
174 public:
175 static OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_wide_vectors);
176 static void restore_live_registers(MacroAssembler* masm, bool restore_wide_vectors = false);
177
178 // Offsets into the register save area
179 // Used by deoptimization when it is managing result register
180 // values on its own
181
182 static int rax_offset_in_bytes(void) { return BytesPerInt * rax_off; }
183 static int rdx_offset_in_bytes(void) { return BytesPerInt * rdx_off; }
184 static int rbx_offset_in_bytes(void) { return BytesPerInt * rbx_off; }
185 static int r15_offset_in_bytes(void) { return BytesPerInt * r15_off; }
186 static int xmm0_offset_in_bytes(void) { return BytesPerInt * xmm0_off; }
187 static int return_offset_in_bytes(void) { return BytesPerInt * return_off; }
188
189 // During deoptimization only the result registers need to be restored,
190 // all the other values have already been extracted.
191 static void restore_result_registers(MacroAssembler* masm);
192 };
193
194 OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_wide_vectors) {
195 int off = 0;
196 int num_xmm_regs = XMMRegister::available_xmm_registers();
197 #if COMPILER2_OR_JVMCI
198 if (save_wide_vectors && UseAVX == 0) {
199 save_wide_vectors = false; // vectors larger than 16 byte long are supported only with AVX
200 }
201 assert(!save_wide_vectors || MaxVectorSize <= 64, "Only up to 64 byte long vectors are supported");
202 #else
203 save_wide_vectors = false; // vectors are generated only by C2 and JVMCI
204 #endif
205
1415 __ movptr(rax, Address(r15_thread, JavaThread::cont_fastpath_offset()));
1416 __ movptr(Address(rsp, ContinuationEntry::parent_cont_fastpath_offset()), rax);
1417 __ movq(rax, Address(r15_thread, JavaThread::held_monitor_count_offset()));
1418 __ movq(Address(rsp, ContinuationEntry::parent_held_monitor_count_offset()), rax);
1419
1420 __ movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), 0);
1421 __ movq(Address(r15_thread, JavaThread::held_monitor_count_offset()), 0);
1422 }
1423
1424 //---------------------------- continuation_enter_cleanup ---------------------------
1425 //
1426 // Arguments:
1427 // rsp: pointer to the ContinuationEntry
1428 //
1429 // Results:
1430 // rsp: pointer to the spilled rbp in the entry frame
1431 //
1432 // Kills:
1433 // rbx
1434 //
1435 static void continuation_enter_cleanup(MacroAssembler* masm) {
1436 #ifdef ASSERT
1437 Label L_good_sp;
1438 __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
1439 __ jcc(Assembler::equal, L_good_sp);
1440 __ stop("Incorrect rsp at continuation_enter_cleanup");
1441 __ bind(L_good_sp);
1442 #endif
1443 __ movptr(rbx, Address(rsp, ContinuationEntry::parent_cont_fastpath_offset()));
1444 __ movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), rbx);
1445
1446 if (CheckJNICalls) {
1447 // Check if this is a virtual thread continuation
1448 Label L_skip_vthread_code;
1449 __ cmpl(Address(rsp, ContinuationEntry::flags_offset()), 0);
1450 __ jcc(Assembler::equal, L_skip_vthread_code);
1451
1452 // If the held monitor count is > 0 and this vthread is terminating then
1453 // it failed to release a JNI monitor. So we issue the same log message
1454 // that JavaThread::exit does.
1455 __ cmpptr(Address(r15_thread, JavaThread::jni_monitor_count_offset()), 0);
1605 address stub = CompiledDirectCall::emit_to_interp_stub(masm, __ pc());
1606 if (stub == nullptr) {
1607 fatal("CodeCache is full at gen_continuation_enter");
1608 }
1609
1610 // The call needs to be resolved. There's a special case for this in
1611 // SharedRuntime::find_callee_info_helper() which calls
1612 // LinkResolver::resolve_continuation_enter() which resolves the call to
1613 // Continuation.enter(Continuation c, boolean isContinue).
1614 __ call(resolve);
1615
1616 oop_maps->add_gc_map(__ pc() - start, map);
1617 __ post_call_nop();
1618
1619 __ jmpb(L_exit);
1620
1621 // --- Thawing path
1622
1623 __ bind(L_thaw);
1624
1625 ContinuationEntry::_thaw_call_pc_offset = __ pc() - start;
1626 __ call(RuntimeAddress(StubRoutines::cont_thaw()));
1627
1628 ContinuationEntry::_return_pc_offset = __ pc() - start;
1629 oop_maps->add_gc_map(__ pc() - start, map->deep_copy());
1630 __ post_call_nop();
1631
1632 // --- Normal exit (resolve/thawing)
1633
1634 __ bind(L_exit);
1635
1636 continuation_enter_cleanup(masm);
1637 __ pop(rbp);
1638 __ ret(0);
1639
1640 // --- Exception handling path
1641
1642 exception_offset = __ pc() - start;
1643
1644 continuation_enter_cleanup(masm);
1645 __ pop(rbp);
1708 continuation_enter_cleanup(masm);
1709 __ pop(rbp);
1710 __ ret(0);
1711
1712 __ bind(L_pinned);
1713
1714 // Pinned, return to caller
1715
1716 // handle pending exception thrown by freeze
1717 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD);
1718 Label ok;
1719 __ jcc(Assembler::equal, ok);
1720 __ leave();
1721 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
1722 __ bind(ok);
1723
1724 __ leave();
1725 __ ret(0);
1726 }
1727
1728 void SharedRuntime::continuation_enter_cleanup(MacroAssembler* masm) {
1729 ::continuation_enter_cleanup(masm);
1730 }
1731
1732 static void gen_special_dispatch(MacroAssembler* masm,
1733 const methodHandle& method,
1734 const BasicType* sig_bt,
1735 const VMRegPair* regs) {
1736 verify_oop_args(masm, method, sig_bt, regs);
1737 vmIntrinsics::ID iid = method->intrinsic_id();
1738
1739 // Now write the args into the outgoing interpreter space
1740 bool has_receiver = false;
1741 Register receiver_reg = noreg;
1742 int member_arg_pos = -1;
1743 Register member_reg = noreg;
1744 int ref_kind = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid);
1745 if (ref_kind != 0) {
1746 member_arg_pos = method->size_of_parameters() - 1; // trailing MemberName argument
1747 member_reg = rbx; // known to be free at this point
1748 has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind);
1749 } else if (iid == vmIntrinsics::_invokeBasic) {
1750 has_receiver = true;
1751 } else if (iid == vmIntrinsics::_linkToNative) {
2275
2276 // Test if the oopMark is an obvious stack pointer, i.e.,
2277 // 1) (mark & 3) == 0, and
2278 // 2) rsp <= mark < mark + os::pagesize()
2279 // These 3 tests can be done by evaluating the following
2280 // expression: ((mark - rsp) & (3 - os::vm_page_size())),
2281 // assuming both stack pointer and pagesize have their
2282 // least significant 2 bits clear.
2283 // NOTE: the oopMark is in swap_reg %rax as the result of cmpxchg
2284
2285 __ subptr(swap_reg, rsp);
2286 __ andptr(swap_reg, 3 - (int)os::vm_page_size());
2287
2288 // Save the test result, for recursive case, the result is zero
2289 __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2290 __ jcc(Assembler::notEqual, slow_path_lock);
2291 } else {
2292 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
2293 __ lightweight_lock(obj_reg, swap_reg, r15_thread, rscratch1, slow_path_lock);
2294 }
2295 __ jmp (lock_done);
2296
2297 __ bind(count_mon);
2298 __ inc_held_monitor_count();
2299
2300 // Slow path will re-enter here
2301 __ bind(lock_done);
2302 }
2303
2304 // Finally just about ready to make the JNI call
2305
2306 // get JNIEnv* which is first argument to native
2307 __ lea(c_rarg0, Address(r15_thread, in_bytes(JavaThread::jni_environment_offset())));
2308
2309 // Now set thread in native
2310 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native);
2311
2312 __ call(RuntimeAddress(native_func));
2313
2314 // Verify or restore cpu control state after JNI call
2315 __ restore_cpu_control_state_after_jni(rscratch1);
2316
2369 // by hand.
2370 //
2371 __ vzeroupper();
2372 save_native_result(masm, ret_type, stack_slots);
2373 __ mov(c_rarg0, r15_thread);
2374 __ mov(r12, rsp); // remember sp
2375 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
2376 __ andptr(rsp, -16); // align stack as required by ABI
2377 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
2378 __ mov(rsp, r12); // restore sp
2379 __ reinit_heapbase();
2380 // Restore any method result value
2381 restore_native_result(masm, ret_type, stack_slots);
2382 __ bind(Continue);
2383 }
2384
2385 // change thread state
2386 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java);
2387 __ bind(after_transition);
2388
2389 // Check preemption for Object.wait()
2390 if (method->is_object_wait0()) {
2391 Label not_preempted;
2392 __ movptr(rscratch1, Address(r15_thread, JavaThread::preempt_alternate_return_offset()));
2393 __ cmpptr(rscratch1, NULL_WORD);
2394 __ jccb(Assembler::equal, not_preempted);
2395 __ movptr(Address(r15_thread, JavaThread::preempt_alternate_return_offset()), NULL_WORD);
2396 __ jmp(rscratch1);
2397 __ bind(not_preempted);
2398 }
2399 int resume_wait_offset = ((intptr_t)__ pc()) - start;
2400
2401 Label reguard;
2402 Label reguard_done;
2403 __ cmpl(Address(r15_thread, JavaThread::stack_guard_state_offset()), StackOverflow::stack_guard_yellow_reserved_disabled);
2404 __ jcc(Assembler::equal, reguard);
2405 __ bind(reguard_done);
2406
2407 // native result if any is live
2408
2409 // Unlock
2410 Label slow_path_unlock;
2411 Label unlock_done;
2412 if (method->is_synchronized()) {
2413
2414 Label fast_done;
2415
2416 // Get locked oop from the handle we passed to jni
2417 __ movptr(obj_reg, Address(oop_handle_reg, 0));
2418
2419 if (LockingMode == LM_LEGACY) {
2420 Label not_recur;
2421 // Simple recursive lock?
2422 __ cmpptr(Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size), NULL_WORD);
2423 __ jcc(Assembler::notEqual, not_recur);
2424 __ jmpb(fast_done);
2425 __ bind(not_recur);
2426 }
2427
2428 // Must save rax if it is live now because cmpxchg must use it
2429 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2430 save_native_result(masm, ret_type, stack_slots);
2431 }
2432
2433 if (LockingMode == LM_MONITOR) {
2434 __ jmp(slow_path_unlock);
2435 } else if (LockingMode == LM_LEGACY) {
2436 // get address of the stack lock
2437 __ lea(rax, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2438 // get old displaced header
2439 __ movptr(old_hdr, Address(rax, 0));
2440
2441 // Atomic swap old header if oop still contains the stack lock
2442 __ lock();
2443 __ cmpxchgptr(old_hdr, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2444 __ jcc(Assembler::notEqual, slow_path_unlock);
2445 __ dec_held_monitor_count();
2446 } else {
2447 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
2448 __ lightweight_unlock(obj_reg, swap_reg, r15_thread, lock_reg, slow_path_unlock);
2449 }
2450
2451 // slow path re-enters here
2452 __ bind(unlock_done);
2453 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2454 restore_native_result(masm, ret_type, stack_slots);
2455 }
2456
2457 __ bind(fast_done);
2458 }
2459 if (DTraceMethodProbes) {
2460 save_native_result(masm, ret_type, stack_slots);
2461 __ mov_metadata(c_rarg1, method());
2462 __ call_VM_leaf(
2463 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
2464 r15_thread, c_rarg1);
2465 restore_native_result(masm, ret_type, stack_slots);
2466 }
2467
2468 __ reset_last_Java_frame(false);
2503 // and forward the exception
2504 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2505
2506 // Slow path locking & unlocking
2507 if (method->is_synchronized()) {
2508
2509 // BEGIN Slow path lock
2510 __ bind(slow_path_lock);
2511
2512 // has last_Java_frame setup. No exceptions so do vanilla call not call_VM
2513 // args are (oop obj, BasicLock* lock, JavaThread* thread)
2514
2515 // protect the args we've loaded
2516 save_args(masm, total_c_args, c_arg, out_regs);
2517
2518 __ mov(c_rarg0, obj_reg);
2519 __ mov(c_rarg1, lock_reg);
2520 __ mov(c_rarg2, r15_thread);
2521
2522 // Not a leaf but we have last_Java_frame setup as we want
2523 // Force freeze slow path on ObjectMonitor::enter() for now which will fail with freeze_pinned_native.
2524 __ push_cont_fastpath();
2525 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3);
2526 __ pop_cont_fastpath();
2527 restore_args(masm, total_c_args, c_arg, out_regs);
2528
2529 #ifdef ASSERT
2530 { Label L;
2531 __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), NULL_WORD);
2532 __ jcc(Assembler::equal, L);
2533 __ stop("no pending exception allowed on exit from monitorenter");
2534 __ bind(L);
2535 }
2536 #endif
2537 __ jmp(lock_done);
2538
2539 // END Slow path lock
2540
2541 // BEGIN Slow path unlock
2542 __ bind(slow_path_unlock);
2543
2544 // If we haven't already saved the native result we must save it now as xmm registers
2545 // are still exposed.
2546 __ vzeroupper();
2598 __ mov(rsp, r12); // restore sp
2599 __ reinit_heapbase();
2600 restore_native_result(masm, ret_type, stack_slots);
2601 // and continue
2602 __ jmp(reguard_done);
2603
2604
2605
2606 __ flush();
2607
2608 nmethod *nm = nmethod::new_native_nmethod(method,
2609 compile_id,
2610 masm->code(),
2611 vep_offset,
2612 frame_complete,
2613 stack_slots / VMRegImpl::slots_per_word,
2614 (is_static ? in_ByteSize(klass_offset) : in_ByteSize(receiver_offset)),
2615 in_ByteSize(lock_slot_offset*VMRegImpl::stack_slot_size),
2616 oop_maps);
2617
2618 if (nm != nullptr && method->is_object_wait0()) {
2619 SharedRuntime::set_native_frame_resume_entry(nm->code_begin() + resume_wait_offset);
2620 }
2621
2622 return nm;
2623 }
2624
2625 // this function returns the adjust size (in number of words) to a c2i adapter
2626 // activation for use during deoptimization
2627 int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals ) {
2628 return (callee_locals - callee_parameters) * Interpreter::stackElementWords;
2629 }
2630
2631
2632 uint SharedRuntime::out_preserve_stack_slots() {
2633 return 0;
2634 }
2635
2636
2637 // Number of stack slots between incoming argument block and the start of
2638 // a new frame. The PROLOG must add this many slots to the stack. The
2639 // EPILOG must remove this many slots. amd64 needs two slots for
2640 // return address.
2641 uint SharedRuntime::in_preserve_stack_slots() {
2642 return 4 + 2 * VerifyStackAtCalls;
2643 }
2644
2645 VMReg SharedRuntime::thread_register() {
2646 return r15_thread->as_VMReg();
2647 }
2648
2649 //------------------------------generate_deopt_blob----------------------------
2650 void SharedRuntime::generate_deopt_blob() {
2651 // Allocate space for the code
2652 ResourceMark rm;
2653 // Setup code generation tools
2654 int pad = 0;
2655 if (UseAVX > 2) {
2656 pad += 1024;
2657 }
2658 if (UseAPX) {
2659 pad += 1024;
2660 }
2661 #if INCLUDE_JVMCI
2662 if (EnableJVMCI) {
2663 pad += 512; // Increase the buffer size when compiling for JVMCI
2664 }
2665 #endif
2666 CodeBuffer buffer("deopt_blob", 2560+pad, 1024);
2667 MacroAssembler* masm = new MacroAssembler(&buffer);
2668 int frame_size_in_words;
|