915 bind(no_reserved_zone_enabling);
916 }
917
918 verify_oop(R17_tos, state);
919
920 merge_frames(/*top_frame_sp*/ R21_sender_SP, /*return_pc*/ R0, R11_scratch1, R12_scratch2);
921 mtlr(R0);
922 pop_cont_fastpath();
923 BLOCK_COMMENT("} remove_activation");
924 }
925
926 // Lock object
927 //
928 // Registers alive
929 // monitor - Address of the BasicObjectLock to be used for locking,
930 // which must be initialized with the object to lock.
931 // object - Address of the object to be locked.
932 //
933 void InterpreterMacroAssembler::lock_object(Register monitor, Register object) {
934 if (LockingMode == LM_MONITOR) {
935 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), monitor);
936 } else {
937 // template code (for LM_LEGACY):
938 //
939 // markWord displaced_header = obj->mark().set_unlocked();
940 // monitor->lock()->set_displaced_header(displaced_header);
941 // if (Atomic::cmpxchg(/*addr*/obj->mark_addr(), /*cmp*/displaced_header, /*ex=*/monitor) == displaced_header) {
942 // // We stored the monitor address into the object's mark word.
943 // } else if (THREAD->is_lock_owned((address)displaced_header))
944 // // Simple recursive case.
945 // monitor->lock()->set_displaced_header(nullptr);
946 // } else {
947 // // Slow path.
948 // InterpreterRuntime::monitorenter(THREAD, monitor);
949 // }
950
951 const Register header = R7_ARG5;
952 const Register object_mark_addr = R8_ARG6;
953 const Register current_header = R9_ARG7;
954 const Register tmp = R10_ARG8;
955
956 Label count_locking, done;
957 Label cas_failed, slow_case;
958
959 assert_different_registers(header, object_mark_addr, current_header, tmp);
960
961 // markWord displaced_header = obj->mark().set_unlocked();
962
963 if (DiagnoseSyncOnValueBasedClasses != 0) {
964 load_klass(tmp, object);
965 lbz(tmp, in_bytes(Klass::misc_flags_offset()), tmp);
966 testbitdi(CCR0, R0, tmp, exact_log2(KlassFlags::_misc_is_value_based_class));
967 bne(CCR0, slow_case);
968 }
969
970 if (LockingMode == LM_LIGHTWEIGHT) {
971 lightweight_lock(monitor, object, header, tmp, slow_case);
972 b(count_locking);
973 } else if (LockingMode == LM_LEGACY) {
974 // Load markWord from object into header.
975 ld(header, oopDesc::mark_offset_in_bytes(), object);
976
977 // Set displaced_header to be (markWord of object | UNLOCK_VALUE).
978 ori(header, header, markWord::unlocked_value);
979
980 // monitor->lock()->set_displaced_header(displaced_header);
981 const int lock_offset = in_bytes(BasicObjectLock::lock_offset());
982 const int mark_offset = lock_offset +
983 BasicLock::displaced_header_offset_in_bytes();
984
985 // Initialize the box (Must happen before we update the object mark!).
986 std(header, mark_offset, monitor);
987
988 // if (Atomic::cmpxchg(/*addr*/obj->mark_addr(), /*cmp*/displaced_header, /*ex=*/monitor) == displaced_header) {
989
990 // Store stack address of the BasicObjectLock (this is monitor) into object.
991 addi(object_mark_addr, object, oopDesc::mark_offset_in_bytes());
992
1018 sub(current_header, current_header, R1_SP);
1019
1020 assert(os::vm_page_size() > 0xfff, "page size too small - change the constant");
1021 load_const_optimized(tmp, ~(os::vm_page_size()-1) | markWord::lock_mask_in_place);
1022
1023 and_(R0/*==0?*/, current_header, tmp);
1024 // If condition is true we are done and hence we can store 0 in the displaced
1025 // header indicating it is a recursive lock.
1026 bne(CCR0, slow_case);
1027 std(R0/*==0!*/, mark_offset, monitor);
1028 b(count_locking);
1029 }
1030
1031 // } else {
1032 // // Slow path.
1033 // InterpreterRuntime::monitorenter(THREAD, monitor);
1034
1035 // None of the above fast optimizations worked so we have to get into the
1036 // slow case of monitor enter.
1037 bind(slow_case);
1038 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), monitor);
1039 b(done);
1040 // }
1041 align(32, 12);
1042 bind(count_locking);
1043 inc_held_monitor_count(current_header /*tmp*/);
1044 bind(done);
1045 }
1046 }
1047
1048 // Unlocks an object. Used in monitorexit bytecode and remove_activation.
1049 //
1050 // Registers alive
1051 // monitor - Address of the BasicObjectLock to be used for locking,
1052 // which must be initialized with the object to lock.
1053 //
1054 // Throw IllegalMonitorException if object is not locked by current thread.
1055 void InterpreterMacroAssembler::unlock_object(Register monitor) {
1056 if (LockingMode == LM_MONITOR) {
1057 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), monitor);
1058 } else {
1059
1060 // template code (for LM_LEGACY):
1061 //
1062 // if ((displaced_header = monitor->displaced_header()) == nullptr) {
1063 // // Recursive unlock. Mark the monitor unlocked by setting the object field to null.
1120 b(free_slot);
1121
1122 // } else {
1123 // // Slow path.
1124 // InterpreterRuntime::monitorexit(monitor);
1125
1126 // The lock has been converted into a heavy lock and hence
1127 // we need to get into the slow case.
1128 bind(slow_case);
1129 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), monitor);
1130 // }
1131
1132 Label done;
1133 b(done); // Monitor register may be overwritten! Runtime has already freed the slot.
1134
1135 // Exchange worked, do monitor->set_obj(nullptr);
1136 align(32, 12);
1137 bind(free_slot);
1138 li(R0, 0);
1139 std(R0, in_bytes(BasicObjectLock::obj_offset()), monitor);
1140 dec_held_monitor_count(current_header /*tmp*/);
1141 bind(done);
1142 }
1143 }
1144
1145 // Load compiled (i2c) or interpreter entry when calling from interpreted and
1146 // do the call. Centralized so that all interpreter calls will do the same actions.
1147 // If jvmti single stepping is on for a thread we must not call compiled code.
1148 //
1149 // Input:
1150 // - Rtarget_method: method to call
1151 // - Rret_addr: return address
1152 // - 2 scratch regs
1153 //
1154 void InterpreterMacroAssembler::call_from_interpreter(Register Rtarget_method, Register Rret_addr,
1155 Register Rscratch1, Register Rscratch2) {
1156 assert_different_registers(Rscratch1, Rscratch2, Rtarget_method, Rret_addr);
1157 // Assume we want to go compiled if available.
1158 const Register Rtarget_addr = Rscratch1;
1159 const Register Rinterp_only = Rscratch2;
1160
2116 beq(CCR0, Ldone);
2117 li(Rtmp, 0);
2118 mr_if_needed(R3, Rexception);
2119 std(Rtmp, thread_(pending_exception)); // Clear exception in thread
2120 if (Interpreter::rethrow_exception_entry() != nullptr) {
2121 // Already got entry address.
2122 load_dispatch_table(Rtmp, (address*)Interpreter::rethrow_exception_entry());
2123 } else {
2124 // Dynamically load entry address.
2125 int simm16_rest = load_const_optimized(Rtmp, &Interpreter::_rethrow_exception_entry, R0, true);
2126 ld(Rtmp, simm16_rest, Rtmp);
2127 }
2128 mtctr(Rtmp);
2129 save_interpreter_state(Rtmp);
2130 bctr();
2131
2132 align(32, 12);
2133 bind(Ldone);
2134 }
2135
2136 void InterpreterMacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions) {
2137 save_interpreter_state(R11_scratch1);
2138
2139 MacroAssembler::call_VM(oop_result, entry_point, false);
2140
2141 restore_interpreter_state(R11_scratch1, /*bcp_and_mdx_only*/ true);
2142
2143 check_and_handle_popframe(R11_scratch1);
2144 check_and_handle_earlyret(R11_scratch1);
2145 // Now check exceptions manually.
2146 if (check_exceptions) {
2147 check_and_forward_exception(R11_scratch1, R12_scratch2);
2148 }
2149 }
2150
2151 void InterpreterMacroAssembler::call_VM(Register oop_result, address entry_point,
2152 Register arg_1, bool check_exceptions) {
2153 // ARG1 is reserved for the thread.
2154 mr_if_needed(R4_ARG2, arg_1);
2155 call_VM(oop_result, entry_point, check_exceptions);
2156 }
2157
2158 void InterpreterMacroAssembler::call_VM(Register oop_result, address entry_point,
2159 Register arg_1, Register arg_2,
2160 bool check_exceptions) {
2161 // ARG1 is reserved for the thread.
2162 mr_if_needed(R4_ARG2, arg_1);
2163 assert(arg_2 != R4_ARG2, "smashed argument");
2164 mr_if_needed(R5_ARG3, arg_2);
2165 call_VM(oop_result, entry_point, check_exceptions);
2166 }
2167
2168 void InterpreterMacroAssembler::call_VM(Register oop_result, address entry_point,
2169 Register arg_1, Register arg_2, Register arg_3,
2170 bool check_exceptions) {
2171 // ARG1 is reserved for the thread.
2172 mr_if_needed(R4_ARG2, arg_1);
2173 assert(arg_2 != R4_ARG2, "smashed argument");
2174 mr_if_needed(R5_ARG3, arg_2);
2175 assert(arg_3 != R4_ARG2 && arg_3 != R5_ARG3, "smashed argument");
2176 mr_if_needed(R6_ARG4, arg_3);
2177 call_VM(oop_result, entry_point, check_exceptions);
|
915 bind(no_reserved_zone_enabling);
916 }
917
918 verify_oop(R17_tos, state);
919
920 merge_frames(/*top_frame_sp*/ R21_sender_SP, /*return_pc*/ R0, R11_scratch1, R12_scratch2);
921 mtlr(R0);
922 pop_cont_fastpath();
923 BLOCK_COMMENT("} remove_activation");
924 }
925
926 // Lock object
927 //
928 // Registers alive
929 // monitor - Address of the BasicObjectLock to be used for locking,
930 // which must be initialized with the object to lock.
931 // object - Address of the object to be locked.
932 //
933 void InterpreterMacroAssembler::lock_object(Register monitor, Register object) {
934 if (LockingMode == LM_MONITOR) {
935 call_VM_preemptable(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), monitor);
936 } else {
937 // template code (for LM_LEGACY):
938 //
939 // markWord displaced_header = obj->mark().set_unlocked();
940 // monitor->lock()->set_displaced_header(displaced_header);
941 // if (Atomic::cmpxchg(/*addr*/obj->mark_addr(), /*cmp*/displaced_header, /*ex=*/monitor) == displaced_header) {
942 // // We stored the monitor address into the object's mark word.
943 // } else if (THREAD->is_lock_owned((address)displaced_header))
944 // // Simple recursive case.
945 // monitor->lock()->set_displaced_header(nullptr);
946 // } else {
947 // // Slow path.
948 // InterpreterRuntime::monitorenter(THREAD, monitor);
949 // }
950
951 const Register header = R7_ARG5;
952 const Register object_mark_addr = R8_ARG6;
953 const Register current_header = R9_ARG7;
954 const Register tmp = R10_ARG8;
955
956 Label count_locking, done, slow_case, cas_failed;
957
958 assert_different_registers(header, object_mark_addr, current_header, tmp);
959
960 // markWord displaced_header = obj->mark().set_unlocked();
961
962 if (DiagnoseSyncOnValueBasedClasses != 0) {
963 load_klass(tmp, object);
964 lbz(tmp, in_bytes(Klass::misc_flags_offset()), tmp);
965 testbitdi(CCR0, R0, tmp, exact_log2(KlassFlags::_misc_is_value_based_class));
966 bne(CCR0, slow_case);
967 }
968
969 if (LockingMode == LM_LIGHTWEIGHT) {
970 lightweight_lock(monitor, object, header, tmp, slow_case);
971 b(done);
972 } else if (LockingMode == LM_LEGACY) {
973 // Load markWord from object into header.
974 ld(header, oopDesc::mark_offset_in_bytes(), object);
975
976 // Set displaced_header to be (markWord of object | UNLOCK_VALUE).
977 ori(header, header, markWord::unlocked_value);
978
979 // monitor->lock()->set_displaced_header(displaced_header);
980 const int lock_offset = in_bytes(BasicObjectLock::lock_offset());
981 const int mark_offset = lock_offset +
982 BasicLock::displaced_header_offset_in_bytes();
983
984 // Initialize the box (Must happen before we update the object mark!).
985 std(header, mark_offset, monitor);
986
987 // if (Atomic::cmpxchg(/*addr*/obj->mark_addr(), /*cmp*/displaced_header, /*ex=*/monitor) == displaced_header) {
988
989 // Store stack address of the BasicObjectLock (this is monitor) into object.
990 addi(object_mark_addr, object, oopDesc::mark_offset_in_bytes());
991
1017 sub(current_header, current_header, R1_SP);
1018
1019 assert(os::vm_page_size() > 0xfff, "page size too small - change the constant");
1020 load_const_optimized(tmp, ~(os::vm_page_size()-1) | markWord::lock_mask_in_place);
1021
1022 and_(R0/*==0?*/, current_header, tmp);
1023 // If condition is true we are done and hence we can store 0 in the displaced
1024 // header indicating it is a recursive lock.
1025 bne(CCR0, slow_case);
1026 std(R0/*==0!*/, mark_offset, monitor);
1027 b(count_locking);
1028 }
1029
1030 // } else {
1031 // // Slow path.
1032 // InterpreterRuntime::monitorenter(THREAD, monitor);
1033
1034 // None of the above fast optimizations worked so we have to get into the
1035 // slow case of monitor enter.
1036 bind(slow_case);
1037 call_VM_preemptable(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), monitor);
1038 // }
1039
1040 if (LockingMode == LM_LEGACY) {
1041 b(done);
1042 align(32, 12);
1043 bind(count_locking);
1044 inc_held_monitor_count(current_header /*tmp*/);
1045 }
1046 bind(done);
1047 }
1048 }
1049
1050 // Unlocks an object. Used in monitorexit bytecode and remove_activation.
1051 //
1052 // Registers alive
1053 // monitor - Address of the BasicObjectLock to be used for locking,
1054 // which must be initialized with the object to lock.
1055 //
1056 // Throw IllegalMonitorException if object is not locked by current thread.
1057 void InterpreterMacroAssembler::unlock_object(Register monitor) {
1058 if (LockingMode == LM_MONITOR) {
1059 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), monitor);
1060 } else {
1061
1062 // template code (for LM_LEGACY):
1063 //
1064 // if ((displaced_header = monitor->displaced_header()) == nullptr) {
1065 // // Recursive unlock. Mark the monitor unlocked by setting the object field to null.
1122 b(free_slot);
1123
1124 // } else {
1125 // // Slow path.
1126 // InterpreterRuntime::monitorexit(monitor);
1127
1128 // The lock has been converted into a heavy lock and hence
1129 // we need to get into the slow case.
1130 bind(slow_case);
1131 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), monitor);
1132 // }
1133
1134 Label done;
1135 b(done); // Monitor register may be overwritten! Runtime has already freed the slot.
1136
1137 // Exchange worked, do monitor->set_obj(nullptr);
1138 align(32, 12);
1139 bind(free_slot);
1140 li(R0, 0);
1141 std(R0, in_bytes(BasicObjectLock::obj_offset()), monitor);
1142 if (LockingMode == LM_LEGACY) {
1143 dec_held_monitor_count(current_header /*tmp*/);
1144 }
1145 bind(done);
1146 }
1147 }
1148
1149 // Load compiled (i2c) or interpreter entry when calling from interpreted and
1150 // do the call. Centralized so that all interpreter calls will do the same actions.
1151 // If jvmti single stepping is on for a thread we must not call compiled code.
1152 //
1153 // Input:
1154 // - Rtarget_method: method to call
1155 // - Rret_addr: return address
1156 // - 2 scratch regs
1157 //
1158 void InterpreterMacroAssembler::call_from_interpreter(Register Rtarget_method, Register Rret_addr,
1159 Register Rscratch1, Register Rscratch2) {
1160 assert_different_registers(Rscratch1, Rscratch2, Rtarget_method, Rret_addr);
1161 // Assume we want to go compiled if available.
1162 const Register Rtarget_addr = Rscratch1;
1163 const Register Rinterp_only = Rscratch2;
1164
2120 beq(CCR0, Ldone);
2121 li(Rtmp, 0);
2122 mr_if_needed(R3, Rexception);
2123 std(Rtmp, thread_(pending_exception)); // Clear exception in thread
2124 if (Interpreter::rethrow_exception_entry() != nullptr) {
2125 // Already got entry address.
2126 load_dispatch_table(Rtmp, (address*)Interpreter::rethrow_exception_entry());
2127 } else {
2128 // Dynamically load entry address.
2129 int simm16_rest = load_const_optimized(Rtmp, &Interpreter::_rethrow_exception_entry, R0, true);
2130 ld(Rtmp, simm16_rest, Rtmp);
2131 }
2132 mtctr(Rtmp);
2133 save_interpreter_state(Rtmp);
2134 bctr();
2135
2136 align(32, 12);
2137 bind(Ldone);
2138 }
2139
2140 void InterpreterMacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions, Label* last_java_pc) {
2141 save_interpreter_state(R11_scratch1);
2142
2143 MacroAssembler::call_VM(oop_result, entry_point, false /*check_exceptions*/, last_java_pc);
2144
2145 restore_interpreter_state(R11_scratch1, /*bcp_and_mdx_only*/ true);
2146
2147 check_and_handle_popframe(R11_scratch1);
2148 check_and_handle_earlyret(R11_scratch1);
2149 // Now check exceptions manually.
2150 if (check_exceptions) {
2151 check_and_forward_exception(R11_scratch1, R12_scratch2);
2152 }
2153 }
2154
2155 void InterpreterMacroAssembler::call_VM(Register oop_result, address entry_point,
2156 Register arg_1, bool check_exceptions) {
2157 // ARG1 is reserved for the thread.
2158 mr_if_needed(R4_ARG2, arg_1);
2159 call_VM(oop_result, entry_point, check_exceptions);
2160 }
2161
2162 void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, address entry_point,
2163 Register arg_1, bool check_exceptions) {
2164 if (!Continuations::enabled()) {
2165 call_VM(oop_result, entry_point, arg_1, check_exceptions);
2166 return;
2167 }
2168
2169 Label resume_pc, not_preempted;
2170
2171 DEBUG_ONLY(ld(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread));
2172 DEBUG_ONLY(cmpdi(CCR0, R0, 0));
2173 asm_assert_eq("Should not have alternate return address set");
2174
2175 // Preserve 2 registers
2176 assert(nonvolatile_accross_vthread_preemtion(R31) && nonvolatile_accross_vthread_preemtion(R22), "");
2177 ld(R3_ARG1, _abi0(callers_sp), R1_SP); // load FP
2178 std(R31, _ijava_state_neg(lresult), R3_ARG1);
2179 std(R22, _ijava_state_neg(fresult), R3_ARG1);
2180
2181 // We set resume_pc as last java pc. It will be saved if the vthread gets preempted.
2182 // Later execution will continue right there.
2183 mr_if_needed(R4_ARG2, arg_1);
2184 push_cont_fastpath();
2185 call_VM(oop_result, entry_point, false /*check_exceptions*/, &resume_pc /* last_java_pc */);
2186 pop_cont_fastpath();
2187
2188 // Jump to handler if the call was preempted
2189 ld(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread);
2190 cmpdi(CCR0, R0, 0);
2191 beq(CCR0, not_preempted);
2192 mtlr(R0);
2193 li(R0, 0);
2194 std(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread);
2195 blr();
2196
2197 bind(resume_pc); // Location to resume execution
2198 restore_after_resume(noreg /* fp */);
2199 bind(not_preempted);
2200 }
2201
2202 void InterpreterMacroAssembler::restore_after_resume(Register fp) {
2203 if (!Continuations::enabled()) return;
2204
2205 const address resume_adapter = TemplateInterpreter::cont_resume_interpreter_adapter();
2206 add_const_optimized(R31, R29_TOC, MacroAssembler::offset_to_global_toc(resume_adapter));
2207 mtctr(R31);
2208 bctrl();
2209 // Restore registers that are preserved across vthread preemption
2210 assert(nonvolatile_accross_vthread_preemtion(R31) && nonvolatile_accross_vthread_preemtion(R22), "");
2211 ld(R3_ARG1, _abi0(callers_sp), R1_SP); // load FP
2212 ld(R31, _ijava_state_neg(lresult), R3_ARG1);
2213 ld(R22, _ijava_state_neg(fresult), R3_ARG1);
2214 #ifdef ASSERT
2215 // Assert FP is in R11_scratch1 (see generate_cont_resume_interpreter_adapter())
2216 {
2217 Label ok;
2218 ld(R12_scratch2, 0, R1_SP); // load fp
2219 cmpd(CCR0, R12_scratch2, R11_scratch1);
2220 beq(CCR0, ok);
2221 stop(FILE_AND_LINE ": FP is expected in R11_scratch1");
2222 bind(ok);
2223 }
2224 #endif
2225 if (fp != noreg && fp != R11_scratch1) {
2226 mr(fp, R11_scratch1);
2227 }
2228 }
2229
2230 void InterpreterMacroAssembler::call_VM(Register oop_result, address entry_point,
2231 Register arg_1, Register arg_2,
2232 bool check_exceptions) {
2233 // ARG1 is reserved for the thread.
2234 mr_if_needed(R4_ARG2, arg_1);
2235 assert(arg_2 != R4_ARG2, "smashed argument");
2236 mr_if_needed(R5_ARG3, arg_2);
2237 call_VM(oop_result, entry_point, check_exceptions);
2238 }
2239
2240 void InterpreterMacroAssembler::call_VM(Register oop_result, address entry_point,
2241 Register arg_1, Register arg_2, Register arg_3,
2242 bool check_exceptions) {
2243 // ARG1 is reserved for the thread.
2244 mr_if_needed(R4_ARG2, arg_1);
2245 assert(arg_2 != R4_ARG2, "smashed argument");
2246 mr_if_needed(R5_ARG3, arg_2);
2247 assert(arg_3 != R4_ARG2 && arg_3 != R5_ARG3, "smashed argument");
2248 mr_if_needed(R6_ARG4, arg_3);
2249 call_VM(oop_result, entry_point, check_exceptions);
|