737 const Register tmp = c_rarg2;
738 const Register obj_reg = c_rarg3; // Will contain the oop
739
740 const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
741 const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
742 const int mark_offset = lock_offset +
743 BasicLock::displaced_header_offset_in_bytes();
744
745 Label slow_case;
746
747 // Load object pointer into obj_reg %c_rarg3
748 ldr(obj_reg, Address(lock_reg, obj_offset));
749
750 if (DiagnoseSyncOnValueBasedClasses != 0) {
751 load_klass(tmp, obj_reg);
752 ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
753 tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
754 br(Assembler::NE, slow_case);
755 }
756
757 if (UseBiasedLocking) {
758 biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp, false, done, &slow_case);
759 }
760
761 // Load (object->mark() | 1) into swap_reg
762 ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
763 orr(swap_reg, rscratch1, 1);
764
765 // Save (object->mark() | 1) into BasicLock's displaced header
766 str(swap_reg, Address(lock_reg, mark_offset));
767
768 assert(lock_offset == 0,
769 "displached header must be first word in BasicObjectLock");
770
771 Label fail;
772 if (PrintBiasedLockingStatistics) {
773 Label fast;
774 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, fast, &fail);
775 bind(fast);
776 atomic_incw(Address((address)BiasedLocking::fast_path_entry_count_addr()),
777 rscratch2, rscratch1, tmp);
778 b(done);
779 bind(fail);
780 } else {
781 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
782 }
783
784 // Fast check for recursive lock.
785 //
786 // Can apply the optimization only if this is a stack lock
787 // allocated in this thread. For efficiency, we can focus on
788 // recently allocated stack locks (instead of reading the stack
789 // base and checking whether 'mark' points inside the current
790 // thread stack):
791 // 1) (mark & 7) == 0, and
792 // 2) sp <= mark < mark + os::pagesize()
793 //
794 // Warning: sp + os::pagesize can overflow the stack base. We must
795 // neither apply the optimization for an inflated lock allocated
796 // just above the thread stack (this is why condition 1 matters)
797 // nor apply the optimization if the stack lock is inside the stack
798 // of another thread. The latter is avoided even in case of overflow
799 // because we have guard pages at the end of all stacks. Hence, if
800 // we go over the stack base and hit the stack of another thread,
801 // this should not be in a writeable area that could contain a
802 // stack lock allocated by that thread. As a consequence, a stack
803 // lock less than page size away from sp is guaranteed to be
804 // owned by the current thread.
805 //
806 // These 3 tests can be done by evaluating the following
807 // expression: ((mark - sp) & (7 - os::vm_page_size())),
808 // assuming both stack pointer and pagesize have their
809 // least significant 3 bits clear.
810 // NOTE: the mark is in swap_reg %r0 as the result of cmpxchg
811 // NOTE2: aarch64 does not like to subtract sp from rn so take a
812 // copy
813 mov(rscratch1, sp);
814 sub(swap_reg, swap_reg, rscratch1);
815 ands(swap_reg, swap_reg, (uint64_t)(7 - os::vm_page_size()));
816
817 // Save the test result, for recursive case, the result is zero
818 str(swap_reg, Address(lock_reg, mark_offset));
819
820 if (PrintBiasedLockingStatistics) {
821 br(Assembler::NE, slow_case);
822 atomic_incw(Address((address)BiasedLocking::fast_path_entry_count_addr()),
823 rscratch2, rscratch1, tmp);
824 }
825 br(Assembler::EQ, done);
826
827 bind(slow_case);
828
829 // Call the runtime routine for slow case
830 call_VM(noreg,
831 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
832 lock_reg);
833
834 bind(done);
835 }
836 }
837
838
839 // Unlocks an object. Used in monitorexit bytecode and
840 // remove_activation. Throws an IllegalMonitorException if object is
841 // not locked by current thread.
842 //
843 // Args:
844 // c_rarg1: BasicObjectLock for lock
845 //
846 // Kills:
847 // r0
848 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
849 // rscratch1, rscratch2 (scratch regs)
850 void InterpreterMacroAssembler::unlock_object(Register lock_reg)
851 {
852 assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");
853
854 if (UseHeavyMonitors) {
855 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
856 } else {
857 Label done;
858
859 const Register swap_reg = r0;
860 const Register header_reg = c_rarg2; // Will contain the old oopMark
861 const Register obj_reg = c_rarg3; // Will contain the oop
862
863 save_bcp(); // Save in case of exception
864
865 // Convert from BasicObjectLock structure to object and BasicLock
866 // structure Store the BasicLock address into %r0
867 lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
868
869 // Load oop into obj_reg(%c_rarg3)
870 ldr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
871
872 // Free entry
873 str(zr, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
874
875 if (UseBiasedLocking) {
876 biased_locking_exit(obj_reg, header_reg, done);
877 }
878
879 // Load the old header from BasicLock structure
880 ldr(header_reg, Address(swap_reg,
881 BasicLock::displaced_header_offset_in_bytes()));
882
883 // Test for recursion
884 cbz(header_reg, done);
885
886 // Atomic swap back the old header
887 cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
888
889 // Call the runtime routine for slow case.
890 str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); // restore obj
891 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
892
893 bind(done);
894
895 restore_bcp();
896 }
897 }
898
899 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
900 Label& zero_continue) {
901 assert(ProfileInterpreter, "must be profiling interpreter");
902 ldr(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
903 cbz(mdp, zero_continue);
904 }
905
906 // Set the method data pointer for the current bcp.
907 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
908 assert(ProfileInterpreter, "must be profiling interpreter");
|
737 const Register tmp = c_rarg2;
738 const Register obj_reg = c_rarg3; // Will contain the oop
739
740 const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
741 const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
742 const int mark_offset = lock_offset +
743 BasicLock::displaced_header_offset_in_bytes();
744
745 Label slow_case;
746
747 // Load object pointer into obj_reg %c_rarg3
748 ldr(obj_reg, Address(lock_reg, obj_offset));
749
750 if (DiagnoseSyncOnValueBasedClasses != 0) {
751 load_klass(tmp, obj_reg);
752 ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
753 tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
754 br(Assembler::NE, slow_case);
755 }
756
757 if (UseFastLocking) {
758 ldr(tmp, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
759 fast_lock(obj_reg, tmp, rscratch1, rscratch2, slow_case);
760 b(done);
761 } else {
762 if (UseBiasedLocking) {
763 biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp, false, done, &slow_case);
764 }
765
766 // Load (object->mark() | 1) into swap_reg
767 ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
768 orr(swap_reg, rscratch1, 1);
769
770 // Save (object->mark() | 1) into BasicLock's displaced header
771 str(swap_reg, Address(lock_reg, mark_offset));
772
773 assert(lock_offset == 0,
774 "displached header must be first word in BasicObjectLock");
775
776 Label fail;
777 if (PrintBiasedLockingStatistics) {
778 Label fast;
779 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, fast, &fail);
780 bind(fast);
781 atomic_incw(Address((address)BiasedLocking::fast_path_entry_count_addr()),
782 rscratch2, rscratch1, tmp);
783 b(done);
784 bind(fail);
785 } else {
786 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
787 }
788
789 // Fast check for recursive lock.
790 //
791 // Can apply the optimization only if this is a stack lock
792 // allocated in this thread. For efficiency, we can focus on
793 // recently allocated stack locks (instead of reading the stack
794 // base and checking whether 'mark' points inside the current
795 // thread stack):
796 // 1) (mark & 7) == 0, and
797 // 2) sp <= mark < mark + os::pagesize()
798 //
799 // Warning: sp + os::pagesize can overflow the stack base. We must
800 // neither apply the optimization for an inflated lock allocated
801 // just above the thread stack (this is why condition 1 matters)
802 // nor apply the optimization if the stack lock is inside the stack
803 // of another thread. The latter is avoided even in case of overflow
804 // because we have guard pages at the end of all stacks. Hence, if
805 // we go over the stack base and hit the stack of another thread,
806 // this should not be in a writeable area that could contain a
807 // stack lock allocated by that thread. As a consequence, a stack
808 // lock less than page size away from sp is guaranteed to be
809 // owned by the current thread.
810 //
811 // These 3 tests can be done by evaluating the following
812 // expression: ((mark - sp) & (7 - os::vm_page_size())),
813 // assuming both stack pointer and pagesize have their
814 // least significant 3 bits clear.
815 // NOTE: the mark is in swap_reg %r0 as the result of cmpxchg
816 // NOTE2: aarch64 does not like to subtract sp from rn so take a
817 // copy
818 mov(rscratch1, sp);
819 sub(swap_reg, swap_reg, rscratch1);
820 ands(swap_reg, swap_reg, (uint64_t)(7 - os::vm_page_size()));
821
822 // Save the test result, for recursive case, the result is zero
823 str(swap_reg, Address(lock_reg, mark_offset));
824
825 if (PrintBiasedLockingStatistics) {
826 br(Assembler::NE, slow_case);
827 atomic_incw(Address((address)BiasedLocking::fast_path_entry_count_addr()),
828 rscratch2, rscratch1, tmp);
829 }
830 br(Assembler::EQ, done);
831 }
832 bind(slow_case);
833
834 // Call the runtime routine for slow case
835 call_VM(noreg,
836 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
837 UseFastLocking ? obj_reg : lock_reg);
838
839 bind(done);
840 }
841 }
842
843
844 // Unlocks an object. Used in monitorexit bytecode and
845 // remove_activation. Throws an IllegalMonitorException if object is
846 // not locked by current thread.
847 //
848 // Args:
849 // c_rarg1: BasicObjectLock for lock
850 //
851 // Kills:
852 // r0
853 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
854 // rscratch1, rscratch2 (scratch regs)
855 void InterpreterMacroAssembler::unlock_object(Register lock_reg)
856 {
857 assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");
858
859 if (UseHeavyMonitors) {
860 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
861 } else {
862 Label done;
863
864 const Register swap_reg = r0;
865 const Register header_reg = c_rarg2; // Will contain the old oopMark
866 const Register obj_reg = c_rarg3; // Will contain the oop
867
868 save_bcp(); // Save in case of exception
869
870 if (UseFastLocking) {
871 Label slow_case;
872
873 // Load oop into obj_reg(%c_rarg3)
874 ldr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
875
876 // Free entry
877 str(zr, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
878
879 // Check for non-symmetric locking. This is allowed by the spec and the interpreter
880 // must handle it.
881 Register tmp = header_reg;
882 ldr(tmp, Address(rthread, JavaThread::lock_stack_current_offset()));
883 ldr(tmp, Address(tmp, -oopSize));
884 cmpoop(tmp, obj_reg);
885 br(Assembler::NE, slow_case);
886
887 ldr(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
888 fast_unlock(obj_reg, header_reg, swap_reg, rscratch1, slow_case);
889 b(done);
890 bind(slow_case);
891 } else {
892 // Convert from BasicObjectLock structure to object and BasicLock
893 // structure Store the BasicLock address into %r0
894 lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
895
896 // Load oop into obj_reg(%c_rarg3)
897 ldr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
898
899 // Free entry
900 str(zr, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
901
902 if (UseBiasedLocking) {
903 biased_locking_exit(obj_reg, header_reg, done);
904 }
905
906 // Load the old header from BasicLock structure
907 ldr(header_reg, Address(swap_reg,
908 BasicLock::displaced_header_offset_in_bytes()));
909
910 // Test for recursion
911 cbz(header_reg, done);
912
913 // Atomic swap back the old header
914 cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
915 }
916 // Call the runtime routine for slow case.
917 str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); // restore obj
918 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
919
920 bind(done);
921
922 restore_bcp();
923 }
924 }
925
926 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
927 Label& zero_continue) {
928 assert(ProfileInterpreter, "must be profiling interpreter");
929 ldr(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
930 cbz(mdp, zero_continue);
931 }
932
933 // Set the method data pointer for the current bcp.
934 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
935 assert(ProfileInterpreter, "must be profiling interpreter");
|