1 /*
2 * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
704 }
705
706 // restore sender esp
707 mov(esp, rscratch2);
708 // remove frame anchor
709 leave();
710 // If we're returning to interpreted code we will shortly be
711 // adjusting SP to allow some space for ESP. If we're returning to
712 // compiled code the saved sender SP was saved in sender_sp, so this
713 // restores it.
714 andr(sp, esp, -16);
715 }
716
717 // Lock object
718 //
719 // Args:
720 // c_rarg1: BasicObjectLock to be used for locking
721 //
722 // Kills:
723 // r0
724 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, .. (param regs)
725 // rscratch1, rscratch2 (scratch regs)
726 void InterpreterMacroAssembler::lock_object(Register lock_reg)
727 {
728 assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1");
729 if (UseHeavyMonitors) {
730 call_VM(noreg,
731 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
732 lock_reg);
733 } else {
734 Label done;
735
736 const Register swap_reg = r0;
737 const Register tmp = c_rarg2;
738 const Register obj_reg = c_rarg3; // Will contain the oop
739
740 const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
741 const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
742 const int mark_offset = lock_offset +
743 BasicLock::displaced_header_offset_in_bytes();
744
745 Label slow_case;
746
747 // Load object pointer into obj_reg %c_rarg3
748 ldr(obj_reg, Address(lock_reg, obj_offset));
749
750 if (DiagnoseSyncOnValueBasedClasses != 0) {
751 load_klass(tmp, obj_reg);
752 ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
753 tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
754 br(Assembler::NE, slow_case);
755 }
756
757 if (UseBiasedLocking) {
758 biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp, false, done, &slow_case);
759 }
760
761 // Load (object->mark() | 1) into swap_reg
762 ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
763 orr(swap_reg, rscratch1, 1);
764
765 // Save (object->mark() | 1) into BasicLock's displaced header
766 str(swap_reg, Address(lock_reg, mark_offset));
767
768 assert(lock_offset == 0,
769 "displached header must be first word in BasicObjectLock");
770
771 Label fail;
772 if (PrintBiasedLockingStatistics) {
773 Label fast;
774 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, fast, &fail);
775 bind(fast);
776 atomic_incw(Address((address)BiasedLocking::fast_path_entry_count_addr()),
777 rscratch2, rscratch1, tmp);
778 b(done);
779 bind(fail);
780 } else {
781 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
782 }
783
784 // Fast check for recursive lock.
785 //
786 // Can apply the optimization only if this is a stack lock
787 // allocated in this thread. For efficiency, we can focus on
788 // recently allocated stack locks (instead of reading the stack
789 // base and checking whether 'mark' points inside the current
790 // thread stack):
791 // 1) (mark & 7) == 0, and
792 // 2) sp <= mark < mark + os::pagesize()
793 //
794 // Warning: sp + os::pagesize can overflow the stack base. We must
795 // neither apply the optimization for an inflated lock allocated
796 // just above the thread stack (this is why condition 1 matters)
797 // nor apply the optimization if the stack lock is inside the stack
798 // of another thread. The latter is avoided even in case of overflow
799 // because we have guard pages at the end of all stacks. Hence, if
800 // we go over the stack base and hit the stack of another thread,
801 // this should not be in a writeable area that could contain a
802 // stack lock allocated by that thread. As a consequence, a stack
803 // lock less than page size away from sp is guaranteed to be
804 // owned by the current thread.
805 //
806 // These 3 tests can be done by evaluating the following
807 // expression: ((mark - sp) & (7 - os::vm_page_size())),
808 // assuming both stack pointer and pagesize have their
809 // least significant 3 bits clear.
810 // NOTE: the mark is in swap_reg %r0 as the result of cmpxchg
811 // NOTE2: aarch64 does not like to subtract sp from rn so take a
812 // copy
813 mov(rscratch1, sp);
814 sub(swap_reg, swap_reg, rscratch1);
815 ands(swap_reg, swap_reg, (uint64_t)(7 - os::vm_page_size()));
816
817 // Save the test result, for recursive case, the result is zero
818 str(swap_reg, Address(lock_reg, mark_offset));
819
820 if (PrintBiasedLockingStatistics) {
821 br(Assembler::NE, slow_case);
822 atomic_incw(Address((address)BiasedLocking::fast_path_entry_count_addr()),
823 rscratch2, rscratch1, tmp);
824 }
825 br(Assembler::EQ, done);
826
827 bind(slow_case);
828
829 // Call the runtime routine for slow case
830 call_VM(noreg,
831 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
832 lock_reg);
833
834 bind(done);
835 }
836 }
837
838
839 // Unlocks an object. Used in monitorexit bytecode and
840 // remove_activation. Throws an IllegalMonitorException if object is
841 // not locked by current thread.
842 //
843 // Args:
844 // c_rarg1: BasicObjectLock for lock
845 //
846 // Kills:
847 // r0
848 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
849 // rscratch1, rscratch2 (scratch regs)
850 void InterpreterMacroAssembler::unlock_object(Register lock_reg)
851 {
852 assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");
853
854 if (UseHeavyMonitors) {
855 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
856 } else {
857 Label done;
858
859 const Register swap_reg = r0;
860 const Register header_reg = c_rarg2; // Will contain the old oopMark
861 const Register obj_reg = c_rarg3; // Will contain the oop
862
863 save_bcp(); // Save in case of exception
864
865 // Convert from BasicObjectLock structure to object and BasicLock
866 // structure Store the BasicLock address into %r0
867 lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
868
869 // Load oop into obj_reg(%c_rarg3)
870 ldr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
871
872 // Free entry
873 str(zr, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
874
875 if (UseBiasedLocking) {
876 biased_locking_exit(obj_reg, header_reg, done);
877 }
878
879 // Load the old header from BasicLock structure
880 ldr(header_reg, Address(swap_reg,
881 BasicLock::displaced_header_offset_in_bytes()));
882
883 // Test for recursion
884 cbz(header_reg, done);
885
886 // Atomic swap back the old header
887 cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
888
889 // Call the runtime routine for slow case.
890 str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); // restore obj
891 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
892
893 bind(done);
894
895 restore_bcp();
896 }
897 }
898
899 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
900 Label& zero_continue) {
901 assert(ProfileInterpreter, "must be profiling interpreter");
902 ldr(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
903 cbz(mdp, zero_continue);
904 }
905
906 // Set the method data pointer for the current bcp.
907 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
908 assert(ProfileInterpreter, "must be profiling interpreter");
|
1 /*
2 * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
704 }
705
706 // restore sender esp
707 mov(esp, rscratch2);
708 // remove frame anchor
709 leave();
710 // If we're returning to interpreted code we will shortly be
711 // adjusting SP to allow some space for ESP. If we're returning to
712 // compiled code the saved sender SP was saved in sender_sp, so this
713 // restores it.
714 andr(sp, esp, -16);
715 }
716
717 // Lock object
718 //
719 // Args:
720 // c_rarg1: BasicObjectLock to be used for locking
721 //
722 // Kills:
723 // r0
724 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, .. (param regs)
725 // rscratch1, rscratch2 (scratch regs)
726 void InterpreterMacroAssembler::lock_object(Register lock_reg)
727 {
728 assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1");
729 if (LockingMode == LM_MONITOR) {
730 call_VM(noreg,
731 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
732 lock_reg);
733 } else {
734 Label done;
735
736 const Register swap_reg = r0;
737 const Register tmp = c_rarg2;
738 const Register obj_reg = c_rarg3; // Will contain the oop
739 const Register tmp2 = c_rarg4;
740 const Register tmp3 = c_rarg5;
741
742 const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
743 const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
744 const int mark_offset = lock_offset +
745 BasicLock::displaced_header_offset_in_bytes();
746
747 Label slow_case;
748
749 // Load object pointer into obj_reg %c_rarg3
750 ldr(obj_reg, Address(lock_reg, obj_offset));
751
752 if (DiagnoseSyncOnValueBasedClasses != 0) {
753 load_klass(tmp, obj_reg);
754 ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
755 tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
756 br(Assembler::NE, slow_case);
757 }
758
759 if (LockingMode == LM_LIGHTWEIGHT) {
760 lightweight_lock(obj_reg, tmp, tmp2, tmp3, slow_case);
761 b(done);
762 } else {
763 if (UseBiasedLocking) {
764 biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp, false, done, &slow_case);
765 }
766
767 // Load (object->mark() | 1) into swap_reg
768 ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
769 orr(swap_reg, rscratch1, 1);
770
771 // Save (object->mark() | 1) into BasicLock's displaced header
772 str(swap_reg, Address(lock_reg, mark_offset));
773
774 assert(lock_offset == 0,
775 "displached header must be first word in BasicObjectLock");
776
777 Label fail;
778 if (PrintBiasedLockingStatistics) {
779 Label fast;
780 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, fast, &fail);
781 bind(fast);
782 atomic_incw(Address((address)BiasedLocking::fast_path_entry_count_addr()),
783 rscratch2, rscratch1, tmp);
784 b(done);
785 bind(fail);
786 } else {
787 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
788 }
789
790 // Fast check for recursive lock.
791 //
792 // Can apply the optimization only if this is a stack lock
793 // allocated in this thread. For efficiency, we can focus on
794 // recently allocated stack locks (instead of reading the stack
795 // base and checking whether 'mark' points inside the current
796 // thread stack):
797 // 1) (mark & 7) == 0, and
798 // 2) sp <= mark < mark + os::pagesize()
799 //
800 // Warning: sp + os::pagesize can overflow the stack base. We must
801 // neither apply the optimization for an inflated lock allocated
802 // just above the thread stack (this is why condition 1 matters)
803 // nor apply the optimization if the stack lock is inside the stack
804 // of another thread. The latter is avoided even in case of overflow
805 // because we have guard pages at the end of all stacks. Hence, if
806 // we go over the stack base and hit the stack of another thread,
807 // this should not be in a writeable area that could contain a
808 // stack lock allocated by that thread. As a consequence, a stack
809 // lock less than page size away from sp is guaranteed to be
810 // owned by the current thread.
811 //
812 // These 3 tests can be done by evaluating the following
813 // expression: ((mark - sp) & (7 - os::vm_page_size())),
814 // assuming both stack pointer and pagesize have their
815 // least significant 3 bits clear.
816 // NOTE: the mark is in swap_reg %r0 as the result of cmpxchg
817 // NOTE2: aarch64 does not like to subtract sp from rn so take a
818 // copy
819 mov(rscratch1, sp);
820 sub(swap_reg, swap_reg, rscratch1);
821 ands(swap_reg, swap_reg, (uint64_t)(7 - os::vm_page_size()));
822
823 // Save the test result, for recursive case, the result is zero
824 str(swap_reg, Address(lock_reg, mark_offset));
825
826 if (PrintBiasedLockingStatistics) {
827 br(Assembler::NE, slow_case);
828 atomic_incw(Address((address)BiasedLocking::fast_path_entry_count_addr()),
829 rscratch2, rscratch1, tmp);
830 }
831 br(Assembler::EQ, done);
832 }
833 bind(slow_case);
834
835 // Call the runtime routine for slow case
836 if (LockingMode == LM_LIGHTWEIGHT) {
837 call_VM(noreg,
838 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter_obj),
839 obj_reg);
840 } else {
841 call_VM(noreg,
842 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
843 lock_reg);
844 }
845
846 bind(done);
847 }
848 }
849
850
851 // Unlocks an object. Used in monitorexit bytecode and
852 // remove_activation. Throws an IllegalMonitorException if object is
853 // not locked by current thread.
854 //
855 // Args:
856 // c_rarg1: BasicObjectLock for lock
857 //
858 // Kills:
859 // r0
860 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
861 // rscratch1, rscratch2 (scratch regs)
862 void InterpreterMacroAssembler::unlock_object(Register lock_reg)
863 {
864 assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");
865
866 if (LockingMode == LM_MONITOR) {
867 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
868 } else {
869 Label done;
870
871 const Register swap_reg = r0;
872 const Register header_reg = c_rarg2; // Will contain the old oopMark
873 const Register obj_reg = c_rarg3; // Will contain the oop
874 const Register tmp_reg = c_rarg4; // Temporary used by lightweight_unlock
875
876 save_bcp(); // Save in case of exception
877
878 if (LockingMode != LM_LIGHTWEIGHT) {
879 // Convert from BasicObjectLock structure to object and BasicLock
880 // structure Store the BasicLock address into %r0
881 lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
882 }
883
884 // Load oop into obj_reg(%c_rarg3)
885 ldr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
886
887 // Free entry
888 str(zr, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
889
890 if (LockingMode == LM_LIGHTWEIGHT) {
891 Label slow_case;
892 lightweight_unlock(obj_reg, header_reg, swap_reg, tmp_reg, slow_case);
893 b(done);
894 bind(slow_case);
895 } else {
896 if (UseBiasedLocking) {
897 biased_locking_exit(obj_reg, header_reg, done);
898 }
899
900 // Load the old header from BasicLock structure
901 ldr(header_reg, Address(swap_reg,
902 BasicLock::displaced_header_offset_in_bytes()));
903
904 // Test for recursion
905 cbz(header_reg, done);
906
907 // Atomic swap back the old header
908 cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
909 }
910 // Call the runtime routine for slow case.
911 str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); // restore obj
912 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
913
914 bind(done);
915
916 restore_bcp();
917 }
918 }
919
920 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
921 Label& zero_continue) {
922 assert(ProfileInterpreter, "must be profiling interpreter");
923 ldr(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
924 cbz(mdp, zero_continue);
925 }
926
927 // Set the method data pointer for the current bcp.
928 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
929 assert(ProfileInterpreter, "must be profiling interpreter");
|