709 leave();
710 // If we're returning to interpreted code we will shortly be
711 // adjusting SP to allow some space for ESP. If we're returning to
712 // compiled code the saved sender SP was saved in sender_sp, so this
713 // restores it.
714 andr(sp, esp, -16);
715 }
716
717 // Lock object
718 //
719 // Args:
720 // c_rarg1: BasicObjectLock to be used for locking
721 //
722 // Kills:
723 // r0
724 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, .. (param regs)
725 // rscratch1, rscratch2 (scratch regs)
726 void InterpreterMacroAssembler::lock_object(Register lock_reg)
727 {
728 assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1");
729 if (UseHeavyMonitors) {
730 call_VM(noreg,
731 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
732 lock_reg);
733 } else {
734 Label done;
735
736 const Register swap_reg = r0;
737 const Register tmp = c_rarg2;
738 const Register obj_reg = c_rarg3; // Will contain the oop
739
740 const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
741 const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
742 const int mark_offset = lock_offset +
743 BasicLock::displaced_header_offset_in_bytes();
744
745 Label slow_case;
746
747 // Load object pointer into obj_reg %c_rarg3
748 ldr(obj_reg, Address(lock_reg, obj_offset));
749
750 if (DiagnoseSyncOnValueBasedClasses != 0) {
751 load_klass(tmp, obj_reg);
752 ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
753 tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
754 br(Assembler::NE, slow_case);
755 }
756
757 if (UseBiasedLocking) {
758 biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp, false, done, &slow_case);
759 }
760
761 // Load (object->mark() | 1) into swap_reg
762 ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
763 orr(swap_reg, rscratch1, 1);
764
765 // Save (object->mark() | 1) into BasicLock's displaced header
766 str(swap_reg, Address(lock_reg, mark_offset));
767
768 assert(lock_offset == 0,
769 "displached header must be first word in BasicObjectLock");
770
771 Label fail;
772 if (PrintBiasedLockingStatistics) {
773 Label fast;
774 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, fast, &fail);
775 bind(fast);
776 atomic_incw(Address((address)BiasedLocking::fast_path_entry_count_addr()),
777 rscratch2, rscratch1, tmp);
778 b(done);
779 bind(fail);
780 } else {
781 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
782 }
783
784 // Fast check for recursive lock.
785 //
786 // Can apply the optimization only if this is a stack lock
787 // allocated in this thread. For efficiency, we can focus on
788 // recently allocated stack locks (instead of reading the stack
789 // base and checking whether 'mark' points inside the current
790 // thread stack):
791 // 1) (mark & 7) == 0, and
792 // 2) sp <= mark < mark + os::pagesize()
793 //
794 // Warning: sp + os::pagesize can overflow the stack base. We must
795 // neither apply the optimization for an inflated lock allocated
796 // just above the thread stack (this is why condition 1 matters)
797 // nor apply the optimization if the stack lock is inside the stack
798 // of another thread. The latter is avoided even in case of overflow
799 // because we have guard pages at the end of all stacks. Hence, if
800 // we go over the stack base and hit the stack of another thread,
801 // this should not be in a writeable area that could contain a
802 // stack lock allocated by that thread. As a consequence, a stack
803 // lock less than page size away from sp is guaranteed to be
804 // owned by the current thread.
805 //
806 // These 3 tests can be done by evaluating the following
807 // expression: ((mark - sp) & (7 - os::vm_page_size())),
808 // assuming both stack pointer and pagesize have their
809 // least significant 3 bits clear.
810 // NOTE: the mark is in swap_reg %r0 as the result of cmpxchg
811 // NOTE2: aarch64 does not like to subtract sp from rn so take a
812 // copy
813 mov(rscratch1, sp);
814 sub(swap_reg, swap_reg, rscratch1);
815 ands(swap_reg, swap_reg, (uint64_t)(7 - os::vm_page_size()));
816
817 // Save the test result, for recursive case, the result is zero
818 str(swap_reg, Address(lock_reg, mark_offset));
819
820 if (PrintBiasedLockingStatistics) {
821 br(Assembler::NE, slow_case);
822 atomic_incw(Address((address)BiasedLocking::fast_path_entry_count_addr()),
823 rscratch2, rscratch1, tmp);
824 }
825 br(Assembler::EQ, done);
826
827 bind(slow_case);
828
829 // Call the runtime routine for slow case
830 call_VM(noreg,
831 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
832 lock_reg);
833
834 bind(done);
835 }
836 }
837
838
839 // Unlocks an object. Used in monitorexit bytecode and
840 // remove_activation. Throws an IllegalMonitorException if object is
841 // not locked by current thread.
842 //
843 // Args:
844 // c_rarg1: BasicObjectLock for lock
845 //
846 // Kills:
847 // r0
848 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
849 // rscratch1, rscratch2 (scratch regs)
850 void InterpreterMacroAssembler::unlock_object(Register lock_reg)
851 {
852 assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");
853
854 if (UseHeavyMonitors) {
855 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
856 } else {
857 Label done;
858
859 const Register swap_reg = r0;
860 const Register header_reg = c_rarg2; // Will contain the old oopMark
861 const Register obj_reg = c_rarg3; // Will contain the oop
862
863 save_bcp(); // Save in case of exception
864
865 // Convert from BasicObjectLock structure to object and BasicLock
866 // structure Store the BasicLock address into %r0
867 lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
868
869 // Load oop into obj_reg(%c_rarg3)
870 ldr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
871
872 // Free entry
873 str(zr, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
874
875 if (UseBiasedLocking) {
876 biased_locking_exit(obj_reg, header_reg, done);
877 }
878
879 // Load the old header from BasicLock structure
880 ldr(header_reg, Address(swap_reg,
881 BasicLock::displaced_header_offset_in_bytes()));
882
883 // Test for recursion
884 cbz(header_reg, done);
885
886 // Atomic swap back the old header
887 cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
888
889 // Call the runtime routine for slow case.
890 str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); // restore obj
891 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
892
893 bind(done);
894
895 restore_bcp();
896 }
897 }
898
899 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
900 Label& zero_continue) {
901 assert(ProfileInterpreter, "must be profiling interpreter");
902 ldr(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
903 cbz(mdp, zero_continue);
904 }
905
906 // Set the method data pointer for the current bcp.
907 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
908 assert(ProfileInterpreter, "must be profiling interpreter");
|
709 leave();
710 // If we're returning to interpreted code we will shortly be
711 // adjusting SP to allow some space for ESP. If we're returning to
712 // compiled code the saved sender SP was saved in sender_sp, so this
713 // restores it.
714 andr(sp, esp, -16);
715 }
716
717 // Lock object
718 //
719 // Args:
720 // c_rarg1: BasicObjectLock to be used for locking
721 //
722 // Kills:
723 // r0
724 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, .. (param regs)
725 // rscratch1, rscratch2 (scratch regs)
726 void InterpreterMacroAssembler::lock_object(Register lock_reg)
727 {
728 assert(lock_reg == c_rarg1, "The argument is only for looks. It must be c_rarg1");
729 if (LockingMode == LM_MONITOR) {
730 call_VM(noreg,
731 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
732 lock_reg);
733 } else {
734 Label done;
735
736 const Register swap_reg = r0;
737 const Register tmp = c_rarg2;
738 const Register obj_reg = c_rarg3; // Will contain the oop
739
740 const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
741 const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
742 const int mark_offset = lock_offset +
743 BasicLock::displaced_header_offset_in_bytes();
744
745 Label slow_case;
746
747 // Load object pointer into obj_reg %c_rarg3
748 ldr(obj_reg, Address(lock_reg, obj_offset));
749
750 if (DiagnoseSyncOnValueBasedClasses != 0) {
751 load_klass(tmp, obj_reg);
752 ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
753 tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
754 br(Assembler::NE, slow_case);
755 }
756
757 if (LockingMode == LM_LIGHTWEIGHT) {
758 ldr(tmp, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
759 fast_lock(obj_reg, tmp, rscratch1, rscratch2, slow_case);
760 b(done);
761 } else {
762 if (UseBiasedLocking) {
763 biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp, false, done, &slow_case);
764 }
765
766 // Load (object->mark() | 1) into swap_reg
767 ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
768 orr(swap_reg, rscratch1, 1);
769
770 // Save (object->mark() | 1) into BasicLock's displaced header
771 str(swap_reg, Address(lock_reg, mark_offset));
772
773 assert(lock_offset == 0,
774 "displached header must be first word in BasicObjectLock");
775
776 Label fail;
777 if (PrintBiasedLockingStatistics) {
778 Label fast;
779 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, fast, &fail);
780 bind(fast);
781 atomic_incw(Address((address)BiasedLocking::fast_path_entry_count_addr()),
782 rscratch2, rscratch1, tmp);
783 b(done);
784 bind(fail);
785 } else {
786 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
787 }
788
789 // Fast check for recursive lock.
790 //
791 // Can apply the optimization only if this is a stack lock
792 // allocated in this thread. For efficiency, we can focus on
793 // recently allocated stack locks (instead of reading the stack
794 // base and checking whether 'mark' points inside the current
795 // thread stack):
796 // 1) (mark & 7) == 0, and
797 // 2) sp <= mark < mark + os::pagesize()
798 //
799 // Warning: sp + os::pagesize can overflow the stack base. We must
800 // neither apply the optimization for an inflated lock allocated
801 // just above the thread stack (this is why condition 1 matters)
802 // nor apply the optimization if the stack lock is inside the stack
803 // of another thread. The latter is avoided even in case of overflow
804 // because we have guard pages at the end of all stacks. Hence, if
805 // we go over the stack base and hit the stack of another thread,
806 // this should not be in a writeable area that could contain a
807 // stack lock allocated by that thread. As a consequence, a stack
808 // lock less than page size away from sp is guaranteed to be
809 // owned by the current thread.
810 //
811 // These 3 tests can be done by evaluating the following
812 // expression: ((mark - sp) & (7 - os::vm_page_size())),
813 // assuming both stack pointer and pagesize have their
814 // least significant 3 bits clear.
815 // NOTE: the mark is in swap_reg %r0 as the result of cmpxchg
816 // NOTE2: aarch64 does not like to subtract sp from rn so take a
817 // copy
818 mov(rscratch1, sp);
819 sub(swap_reg, swap_reg, rscratch1);
820 ands(swap_reg, swap_reg, (uint64_t)(7 - os::vm_page_size()));
821
822 // Save the test result, for recursive case, the result is zero
823 str(swap_reg, Address(lock_reg, mark_offset));
824
825 if (PrintBiasedLockingStatistics) {
826 br(Assembler::NE, slow_case);
827 atomic_incw(Address((address)BiasedLocking::fast_path_entry_count_addr()),
828 rscratch2, rscratch1, tmp);
829 }
830 br(Assembler::EQ, done);
831 }
832 bind(slow_case);
833
834 // Call the runtime routine for slow case
835 if (LockingMode == LM_LIGHTWEIGHT) {
836 call_VM(noreg,
837 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter_obj),
838 obj_reg);
839 } else {
840 call_VM(noreg,
841 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
842 lock_reg);
843 }
844
845 bind(done);
846 }
847 }
848
849
850 // Unlocks an object. Used in monitorexit bytecode and
851 // remove_activation. Throws an IllegalMonitorException if object is
852 // not locked by current thread.
853 //
854 // Args:
855 // c_rarg1: BasicObjectLock for lock
856 //
857 // Kills:
858 // r0
859 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
860 // rscratch1, rscratch2 (scratch regs)
861 void InterpreterMacroAssembler::unlock_object(Register lock_reg)
862 {
863 assert(lock_reg == c_rarg1, "The argument is only for looks. It must be rarg1");
864
865 if (LockingMode == LM_MONITOR) {
866 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
867 } else {
868 Label done;
869
870 const Register swap_reg = r0;
871 const Register header_reg = c_rarg2; // Will contain the old oopMark
872 const Register obj_reg = c_rarg3; // Will contain the oop
873
874 save_bcp(); // Save in case of exception
875
876 if (LockingMode != LM_LIGHTWEIGHT) {
877 // Convert from BasicObjectLock structure to object and BasicLock
878 // structure Store the BasicLock address into %r0
879 lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
880 }
881
882 // Load oop into obj_reg(%c_rarg3)
883 ldr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
884
885 // Free entry
886 str(zr, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
887
888 if (LockingMode == LM_LIGHTWEIGHT) {
889 Label slow_case;
890
891 // Check for non-symmetric locking. This is allowed by the spec and the interpreter
892 // must handle it.
893 Register tmp = rscratch1;
894 // First check for lock-stack underflow.
895 ldrw(tmp, Address(rthread, JavaThread::lock_stack_top_offset()));
896 cmpw(tmp, (unsigned)LockStack::start_offset());
897 br(Assembler::LE, slow_case);
898 // Then check if the top of the lock-stack matches the unlocked object.
899 subw(tmp, tmp, oopSize);
900 ldr(tmp, Address(rthread, tmp));
901 cmpoop(tmp, obj_reg);
902 br(Assembler::NE, slow_case);
903
904 ldr(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
905 tbnz(header_reg, exact_log2(markWord::monitor_value), slow_case);
906 fast_unlock(obj_reg, header_reg, swap_reg, rscratch1, slow_case);
907 b(done);
908 bind(slow_case);
909 } else {
910 if (UseBiasedLocking) {
911 biased_locking_exit(obj_reg, header_reg, done);
912 }
913
914 // Load the old header from BasicLock structure
915 ldr(header_reg, Address(swap_reg,
916 BasicLock::displaced_header_offset_in_bytes()));
917
918 // Test for recursion
919 cbz(header_reg, done);
920
921 // Atomic swap back the old header
922 cmpxchg_obj_header(swap_reg, header_reg, obj_reg, rscratch1, done, /*fallthrough*/NULL);
923 }
924 // Call the runtime routine for slow case.
925 str(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes())); // restore obj
926 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
927
928 bind(done);
929
930 restore_bcp();
931 }
932 }
933
934 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
935 Label& zero_continue) {
936 assert(ProfileInterpreter, "must be profiling interpreter");
937 ldr(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
938 cbz(mdp, zero_continue);
939 }
940
941 // Set the method data pointer for the current bcp.
942 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
943 assert(ProfileInterpreter, "must be profiling interpreter");
|