649 // If we're returning to interpreted code we will shortly be
650 // adjusting SP to allow some space for ESP. If we're returning to
651 // compiled code the saved sender SP was saved in sender_sp, so this
652 // restores it.
653 andr(sp, esp, -16);
654 }
655
656 // Lock object
657 //
658 // Args:
659 // c_rarg1: BasicObjectLock to be used for locking
660 //
661 // Kills:
662 // r0
663 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, .. (param regs)
664 // rscratch1, rscratch2 (scratch regs)
665 void InterpreterMacroAssembler::lock_object(Register lock_reg)
666 {
667 assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1");
668 if (LockingMode == LM_MONITOR) {
669 call_VM(noreg,
670 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
671 lock_reg);
672 } else {
673 Label count, done;
674
675 const Register swap_reg = r0;
676 const Register tmp = c_rarg2;
677 const Register obj_reg = c_rarg3; // Will contain the oop
678 const Register tmp2 = c_rarg4;
679 const Register tmp3 = c_rarg5;
680
681 const int obj_offset = in_bytes(BasicObjectLock::obj_offset());
682 const int lock_offset = in_bytes(BasicObjectLock::lock_offset());
683 const int mark_offset = lock_offset +
684 BasicLock::displaced_header_offset_in_bytes();
685
686 Label slow_case;
687
688 // Load object pointer into obj_reg %c_rarg3
689 ldr(obj_reg, Address(lock_reg, obj_offset));
690
691 if (DiagnoseSyncOnValueBasedClasses != 0) {
692 load_klass(tmp, obj_reg);
693 ldrb(tmp, Address(tmp, Klass::misc_flags_offset()));
694 tst(tmp, KlassFlags::_misc_is_value_based_class);
695 br(Assembler::NE, slow_case);
696 }
697
698 if (LockingMode == LM_LIGHTWEIGHT) {
699 lightweight_lock(lock_reg, obj_reg, tmp, tmp2, tmp3, slow_case);
700 b(count);
701 } else if (LockingMode == LM_LEGACY) {
702 // Load (object->mark() | 1) into swap_reg
703 ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
704 orr(swap_reg, rscratch1, 1);
705
706 // Save (object->mark() | 1) into BasicLock's displaced header
707 str(swap_reg, Address(lock_reg, mark_offset));
708
709 assert(lock_offset == 0,
710 "displached header must be first word in BasicObjectLock");
711
712 Label fail;
713 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, count, /*fallthrough*/nullptr);
714
715 // Fast check for recursive lock.
716 //
717 // Can apply the optimization only if this is a stack lock
718 // allocated in this thread. For efficiency, we can focus on
719 // recently allocated stack locks (instead of reading the stack
720 // base and checking whether 'mark' points inside the current
730 // because we have guard pages at the end of all stacks. Hence, if
731 // we go over the stack base and hit the stack of another thread,
732 // this should not be in a writeable area that could contain a
733 // stack lock allocated by that thread. As a consequence, a stack
734 // lock less than page size away from sp is guaranteed to be
735 // owned by the current thread.
736 //
737 // These 3 tests can be done by evaluating the following
738 // expression: ((mark - sp) & (7 - os::vm_page_size())),
739 // assuming both stack pointer and pagesize have their
740 // least significant 3 bits clear.
741 // NOTE: the mark is in swap_reg %r0 as the result of cmpxchg
742 // NOTE2: aarch64 does not like to subtract sp from rn so take a
743 // copy
744 mov(rscratch1, sp);
745 sub(swap_reg, swap_reg, rscratch1);
746 ands(swap_reg, swap_reg, (uint64_t)(7 - (int)os::vm_page_size()));
747
748 // Save the test result, for recursive case, the result is zero
749 str(swap_reg, Address(lock_reg, mark_offset));
750 br(Assembler::EQ, count);
751 }
752 bind(slow_case);
753
754 // Call the runtime routine for slow case
755 call_VM(noreg,
756 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
757 lock_reg);
758 b(done);
759
760 bind(count);
761 increment(Address(rthread, JavaThread::held_monitor_count_offset()));
762
763 bind(done);
764 }
765 }
766
767
768 // Unlocks an object. Used in monitorexit bytecode and
769 // remove_activation. Throws an IllegalMonitorException if object is
770 // not locked by current thread.
771 //
772 // Args:
773 // c_rarg1: BasicObjectLock for lock
774 //
775 // Kills:
776 // r0
777 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
778 // rscratch1, rscratch2 (scratch regs)
779 void InterpreterMacroAssembler::unlock_object(Register lock_reg)
780 {
781 assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");
787
788 const Register swap_reg = r0;
789 const Register header_reg = c_rarg2; // Will contain the old oopMark
790 const Register obj_reg = c_rarg3; // Will contain the oop
791 const Register tmp_reg = c_rarg4; // Temporary used by lightweight_unlock
792
793 save_bcp(); // Save in case of exception
794
795 if (LockingMode != LM_LIGHTWEIGHT) {
796 // Convert from BasicObjectLock structure to object and BasicLock
797 // structure Store the BasicLock address into %r0
798 lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset()));
799 }
800
801 // Load oop into obj_reg(%c_rarg3)
802 ldr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset()));
803
804 // Free entry
805 str(zr, Address(lock_reg, BasicObjectLock::obj_offset()));
806
807 if (LockingMode == LM_LIGHTWEIGHT) {
808 Label slow_case;
809 lightweight_unlock(obj_reg, header_reg, swap_reg, tmp_reg, slow_case);
810 b(count);
811 bind(slow_case);
812 } else if (LockingMode == LM_LEGACY) {
813 // Load the old header from BasicLock structure
814 ldr(header_reg, Address(swap_reg,
815 BasicLock::displaced_header_offset_in_bytes()));
816
817 // Test for recursion
818 cbz(header_reg, count);
819
820 // Atomic swap back the old header
821 cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, count, /*fallthrough*/nullptr);
822 }
823 // Call the runtime routine for slow case.
824 str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset())); // restore obj
825 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
826 b(done);
827
828 bind(count);
829 decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
830
831 bind(done);
832 restore_bcp();
833 }
834 }
835
836 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
837 Label& zero_continue) {
838 assert(ProfileInterpreter, "must be profiling interpreter");
839 ldr(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
840 cbz(mdp, zero_continue);
841 }
842
843 // Set the method data pointer for the current bcp.
844 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
845 assert(ProfileInterpreter, "must be profiling interpreter");
846 Label set_mdp;
847 stp(r0, r1, Address(pre(sp, -2 * wordSize)));
848
849 // Test MDO to avoid the call if it is null.
850 ldr(r0, Address(rmethod, in_bytes(Method::method_data_offset())));
1514 save_bcp();
1515 #ifdef ASSERT
1516 {
1517 Label L;
1518 ldr(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
1519 cbz(rscratch1, L);
1520 stop("InterpreterMacroAssembler::call_VM_base:"
1521 " last_sp != nullptr");
1522 bind(L);
1523 }
1524 #endif /* ASSERT */
1525 // super call
1526 MacroAssembler::call_VM_base(oop_result, noreg, last_java_sp,
1527 entry_point, number_of_arguments,
1528 check_exceptions);
1529 // interpreter specific
1530 restore_bcp();
1531 restore_locals();
1532 }
1533
1534 void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr) {
1535 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
1536 Label update, next, none;
1537
1538 verify_oop(obj);
1539
1540 cbnz(obj, update);
1541 orptr(mdo_addr, TypeEntries::null_seen);
1542 b(next);
1543
1544 bind(update);
1545 load_klass(obj, obj);
1546
1547 ldr(rscratch1, mdo_addr);
1548 eor(obj, obj, rscratch1);
1549 tst(obj, TypeEntries::type_klass_mask);
1550 br(Assembler::EQ, next); // klass seen before, nothing to
1551 // do. The unknown bit may have been
1552 // set already but no need to check.
1553
|
649 // If we're returning to interpreted code we will shortly be
650 // adjusting SP to allow some space for ESP. If we're returning to
651 // compiled code the saved sender SP was saved in sender_sp, so this
652 // restores it.
653 andr(sp, esp, -16);
654 }
655
656 // Lock object
657 //
658 // Args:
659 // c_rarg1: BasicObjectLock to be used for locking
660 //
661 // Kills:
662 // r0
663 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, .. (param regs)
664 // rscratch1, rscratch2 (scratch regs)
665 void InterpreterMacroAssembler::lock_object(Register lock_reg)
666 {
667 assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1");
668 if (LockingMode == LM_MONITOR) {
669 call_VM_preemptable(noreg,
670 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
671 lock_reg);
672 } else {
673 Label count, done;
674
675 const Register swap_reg = r0;
676 const Register tmp = c_rarg2;
677 const Register obj_reg = c_rarg3; // Will contain the oop
678 const Register tmp2 = c_rarg4;
679 const Register tmp3 = c_rarg5;
680
681 const int obj_offset = in_bytes(BasicObjectLock::obj_offset());
682 const int lock_offset = in_bytes(BasicObjectLock::lock_offset());
683 const int mark_offset = lock_offset +
684 BasicLock::displaced_header_offset_in_bytes();
685
686 Label slow_case;
687
688 // Load object pointer into obj_reg %c_rarg3
689 ldr(obj_reg, Address(lock_reg, obj_offset));
690
691 if (DiagnoseSyncOnValueBasedClasses != 0) {
692 load_klass(tmp, obj_reg);
693 ldrb(tmp, Address(tmp, Klass::misc_flags_offset()));
694 tst(tmp, KlassFlags::_misc_is_value_based_class);
695 br(Assembler::NE, slow_case);
696 }
697
698 if (LockingMode == LM_LIGHTWEIGHT) {
699 lightweight_lock(lock_reg, obj_reg, tmp, tmp2, tmp3, slow_case);
700 b(done);
701 } else if (LockingMode == LM_LEGACY) {
702 // Load (object->mark() | 1) into swap_reg
703 ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
704 orr(swap_reg, rscratch1, 1);
705
706 // Save (object->mark() | 1) into BasicLock's displaced header
707 str(swap_reg, Address(lock_reg, mark_offset));
708
709 assert(lock_offset == 0,
710 "displached header must be first word in BasicObjectLock");
711
712 Label fail;
713 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, count, /*fallthrough*/nullptr);
714
715 // Fast check for recursive lock.
716 //
717 // Can apply the optimization only if this is a stack lock
718 // allocated in this thread. For efficiency, we can focus on
719 // recently allocated stack locks (instead of reading the stack
720 // base and checking whether 'mark' points inside the current
730 // because we have guard pages at the end of all stacks. Hence, if
731 // we go over the stack base and hit the stack of another thread,
732 // this should not be in a writeable area that could contain a
733 // stack lock allocated by that thread. As a consequence, a stack
734 // lock less than page size away from sp is guaranteed to be
735 // owned by the current thread.
736 //
737 // These 3 tests can be done by evaluating the following
738 // expression: ((mark - sp) & (7 - os::vm_page_size())),
739 // assuming both stack pointer and pagesize have their
740 // least significant 3 bits clear.
741 // NOTE: the mark is in swap_reg %r0 as the result of cmpxchg
742 // NOTE2: aarch64 does not like to subtract sp from rn so take a
743 // copy
744 mov(rscratch1, sp);
745 sub(swap_reg, swap_reg, rscratch1);
746 ands(swap_reg, swap_reg, (uint64_t)(7 - (int)os::vm_page_size()));
747
748 // Save the test result, for recursive case, the result is zero
749 str(swap_reg, Address(lock_reg, mark_offset));
750 br(Assembler::NE, slow_case);
751
752 bind(count);
753 inc_held_monitor_count(rscratch1);
754 b(done);
755 }
756 bind(slow_case);
757
758 // Call the runtime routine for slow case
759 call_VM_preemptable(noreg,
760 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
761 lock_reg);
762
763 bind(done);
764 }
765 }
766
767
768 // Unlocks an object. Used in monitorexit bytecode and
769 // remove_activation. Throws an IllegalMonitorException if object is
770 // not locked by current thread.
771 //
772 // Args:
773 // c_rarg1: BasicObjectLock for lock
774 //
775 // Kills:
776 // r0
777 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
778 // rscratch1, rscratch2 (scratch regs)
779 void InterpreterMacroAssembler::unlock_object(Register lock_reg)
780 {
781 assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");
787
788 const Register swap_reg = r0;
789 const Register header_reg = c_rarg2; // Will contain the old oopMark
790 const Register obj_reg = c_rarg3; // Will contain the oop
791 const Register tmp_reg = c_rarg4; // Temporary used by lightweight_unlock
792
793 save_bcp(); // Save in case of exception
794
795 if (LockingMode != LM_LIGHTWEIGHT) {
796 // Convert from BasicObjectLock structure to object and BasicLock
797 // structure Store the BasicLock address into %r0
798 lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset()));
799 }
800
801 // Load oop into obj_reg(%c_rarg3)
802 ldr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset()));
803
804 // Free entry
805 str(zr, Address(lock_reg, BasicObjectLock::obj_offset()));
806
807 Label slow_case;
808 if (LockingMode == LM_LIGHTWEIGHT) {
809 lightweight_unlock(obj_reg, header_reg, swap_reg, tmp_reg, slow_case);
810 b(done);
811 } else if (LockingMode == LM_LEGACY) {
812 // Load the old header from BasicLock structure
813 ldr(header_reg, Address(swap_reg,
814 BasicLock::displaced_header_offset_in_bytes()));
815
816 // Test for recursion
817 cbz(header_reg, count);
818
819 // Atomic swap back the old header
820 cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, count, &slow_case);
821
822 bind(count);
823 dec_held_monitor_count(rscratch1);
824 b(done);
825 }
826
827 bind(slow_case);
828 // Call the runtime routine for slow case.
829 str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset())); // restore obj
830 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
831 bind(done);
832 restore_bcp();
833 }
834 }
835
836 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
837 Label& zero_continue) {
838 assert(ProfileInterpreter, "must be profiling interpreter");
839 ldr(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
840 cbz(mdp, zero_continue);
841 }
842
843 // Set the method data pointer for the current bcp.
844 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
845 assert(ProfileInterpreter, "must be profiling interpreter");
846 Label set_mdp;
847 stp(r0, r1, Address(pre(sp, -2 * wordSize)));
848
849 // Test MDO to avoid the call if it is null.
850 ldr(r0, Address(rmethod, in_bytes(Method::method_data_offset())));
1514 save_bcp();
1515 #ifdef ASSERT
1516 {
1517 Label L;
1518 ldr(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
1519 cbz(rscratch1, L);
1520 stop("InterpreterMacroAssembler::call_VM_base:"
1521 " last_sp != nullptr");
1522 bind(L);
1523 }
1524 #endif /* ASSERT */
1525 // super call
1526 MacroAssembler::call_VM_base(oop_result, noreg, last_java_sp,
1527 entry_point, number_of_arguments,
1528 check_exceptions);
1529 // interpreter specific
1530 restore_bcp();
1531 restore_locals();
1532 }
1533
1534 void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result,
1535 address entry_point,
1536 Register arg_1) {
1537 assert(arg_1 == c_rarg1, "");
1538 Label resume_pc, not_preempted;
1539
1540 #ifdef ASSERT
1541 {
1542 Label L;
1543 ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset()));
1544 cbz(rscratch1, L);
1545 stop("Should not have alternate return address set");
1546 bind(L);
1547 }
1548 #endif /* ASSERT */
1549
1550 // Force freeze slow path.
1551 push_cont_fastpath();
1552
1553 // Make VM call. In case of preemption set last_pc to the one we want to resume to.
1554 adr(rscratch1, resume_pc);
1555 str(rscratch1, Address(rthread, JavaThread::last_Java_pc_offset()));
1556 call_VM_base(oop_result, noreg, noreg, entry_point, 1, false /*check_exceptions*/);
1557
1558 pop_cont_fastpath();
1559
1560 // Check if preempted.
1561 ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset()));
1562 cbz(rscratch1, not_preempted);
1563 str(zr, Address(rthread, JavaThread::preempt_alternate_return_offset()));
1564 br(rscratch1);
1565
1566 // In case of preemption, this is where we will resume once we finally acquire the monitor.
1567 bind(resume_pc);
1568 restore_after_resume(false /* is_native */);
1569
1570 bind(not_preempted);
1571 }
1572
1573 void InterpreterMacroAssembler::restore_after_resume(bool is_native) {
1574 lea(rscratch1, ExternalAddress(Interpreter::cont_resume_interpreter_adapter()));
1575 blr(rscratch1);
1576 if (is_native) {
1577 // On resume we need to set up stack as expected
1578 push(dtos);
1579 push(ltos);
1580 }
1581 }
1582
1583 void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr) {
1584 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
1585 Label update, next, none;
1586
1587 verify_oop(obj);
1588
1589 cbnz(obj, update);
1590 orptr(mdo_addr, TypeEntries::null_seen);
1591 b(next);
1592
1593 bind(update);
1594 load_klass(obj, obj);
1595
1596 ldr(rscratch1, mdo_addr);
1597 eor(obj, obj, rscratch1);
1598 tst(obj, TypeEntries::type_klass_mask);
1599 br(Assembler::EQ, next); // klass seen before, nothing to
1600 // do. The unknown bit may have been
1601 // set already but no need to check.
1602
|