15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "compiler/compiler_globals.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/barrierSetAssembler.hpp"
30 #include "interp_masm_aarch64.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interpreterRuntime.hpp"
33 #include "logging/log.hpp"
34 #include "oops/arrayOop.hpp"
35 #include "oops/markWord.hpp"
36 #include "oops/method.hpp"
37 #include "oops/methodData.hpp"
38 #include "oops/resolvedFieldEntry.hpp"
39 #include "oops/resolvedIndyEntry.hpp"
40 #include "oops/resolvedMethodEntry.hpp"
41 #include "prims/jvmtiExport.hpp"
42 #include "prims/jvmtiThreadState.hpp"
43 #include "runtime/basicLock.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/javaThread.hpp"
46 #include "runtime/safepointMechanism.hpp"
47 #include "runtime/sharedRuntime.hpp"
48 #include "utilities/powerOfTwo.hpp"
49
50 void InterpreterMacroAssembler::narrow(Register result) {
51
52 // Get method->_constMethod->_result_type
53 ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
54 ldr(rscratch1, Address(rscratch1, Method::const_offset()));
55 ldrb(rscratch1, Address(rscratch1, ConstMethod::result_type_offset()));
56
57 Label done, notBool, notByte, notChar;
191 ldrw(index, Address(rbcp, bcp_offset));
192 } else if (index_size == sizeof(u1)) {
193 load_unsigned_byte(index, Address(rbcp, bcp_offset));
194 } else {
195 ShouldNotReachHere();
196 }
197 }
198
199 void InterpreterMacroAssembler::get_method_counters(Register method,
200 Register mcs, Label& skip) {
201 Label has_counters;
202 ldr(mcs, Address(method, Method::method_counters_offset()));
203 cbnz(mcs, has_counters);
204 call_VM(noreg, CAST_FROM_FN_PTR(address,
205 InterpreterRuntime::build_method_counters), method);
206 ldr(mcs, Address(method, Method::method_counters_offset()));
207 cbz(mcs, skip); // No MethodCounters allocated, OutOfMemory
208 bind(has_counters);
209 }
210
211 // Load object from cpool->resolved_references(index)
212 void InterpreterMacroAssembler::load_resolved_reference_at_index(
213 Register result, Register index, Register tmp) {
214 assert_different_registers(result, index);
215
216 get_constant_pool(result);
217 // load pointer for resolved_references[] objArray
218 ldr(result, Address(result, ConstantPool::cache_offset()));
219 ldr(result, Address(result, ConstantPoolCache::resolved_references_offset()));
220 resolve_oop_handle(result, tmp, rscratch2);
221 // Add in the index
222 add(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
223 load_heap_oop(result, Address(result, index, Address::uxtw(LogBytesPerHeapOop)), tmp, rscratch2);
224 }
225
226 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
227 Register cpool, Register index, Register klass, Register temp) {
228 add(temp, cpool, index, LSL, LogBytesPerWord);
229 ldrh(temp, Address(temp, sizeof(ConstantPool))); // temp = resolved_klass_index
230 ldr(klass, Address(cpool, ConstantPool::resolved_klasses_offset())); // klass = cpool->_resolved_klasses
231 add(klass, klass, temp, LSL, LogBytesPerWord);
232 ldr(klass, Address(klass, Array<Klass*>::base_offset_in_bytes()));
233 }
234
235 // Generate a subtype check: branch to ok_is_subtype if sub_klass is a
236 // subtype of super_klass.
237 //
238 // Args:
239 // r0: superklass
240 // Rsub_klass: subklass
241 //
242 // Kills:
243 // r2, r5
244 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
245 Label& ok_is_subtype) {
246 assert(Rsub_klass != r0, "r0 holds superklass");
247 assert(Rsub_klass != r2, "r2 holds 2ndary super array length");
248 assert(Rsub_klass != r5, "r5 holds 2ndary super array scan ptr");
249
250 // Profile the not-null value's klass.
251 profile_typecheck(r2, Rsub_klass, r5); // blows r2, reloads r5
252
253 // Do the check.
254 check_klass_subtype(Rsub_klass, r0, r2, ok_is_subtype); // blows r2
255 }
256
257 // Java Expression Stack
258
259 void InterpreterMacroAssembler::pop_ptr(Register r) {
260 ldr(r, post(esp, wordSize));
261 }
262
263 void InterpreterMacroAssembler::pop_i(Register r) {
264 ldrw(r, post(esp, wordSize));
265 }
266
267 void InterpreterMacroAssembler::pop_l(Register r) {
268 ldr(r, post(esp, 2 * Interpreter::stackElementSize));
269 }
270
271 void InterpreterMacroAssembler::push_ptr(Register r) {
607 br(Assembler::AL, fast_path);
608 bind(slow_path);
609 push(state);
610 set_last_Java_frame(esp, rfp, pc(), rscratch1);
611 super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), rthread);
612 reset_last_Java_frame(true);
613 pop(state);
614 bind(fast_path);
615
616 // JVMTI support. Make sure the safepoint poll test is issued prior.
617 if (notify_jvmdi) {
618 notify_method_exit(state, NotifyJVMTI); // preserve TOSCA
619 } else {
620 notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
621 }
622
623 // remove activation
624 // get sender esp
625 ldr(rscratch2,
626 Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
627 if (StackReservedPages > 0) {
628 // testing if reserved zone needs to be re-enabled
629 Label no_reserved_zone_enabling;
630
631 // check if already enabled - if so no re-enabling needed
632 assert(sizeof(StackOverflow::StackGuardState) == 4, "unexpected size");
633 ldrw(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
634 cmpw(rscratch1, (u1)StackOverflow::stack_guard_enabled);
635 br(Assembler::EQ, no_reserved_zone_enabling);
636
637 // look for an overflow into the stack reserved zone, i.e.
638 // interpreter_frame_sender_sp <= JavaThread::reserved_stack_activation
639 ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset()));
640 cmp(rscratch2, rscratch1);
641 br(Assembler::LS, no_reserved_zone_enabling);
642
643 JFR_ONLY(leave_jfr_critical_section();)
644
645 call_VM_leaf(
646 CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), rthread);
647 call_VM(noreg, CAST_FROM_FN_PTR(address,
648 InterpreterRuntime::throw_delayed_StackOverflowError));
649 should_not_reach_here();
650
651 bind(no_reserved_zone_enabling);
652 }
653
654 // remove frame anchor
655 leave();
656
657 JFR_ONLY(leave_jfr_critical_section();)
658
659 // restore sender esp
660 mov(esp, rscratch2);
661
662 // If we're returning to interpreted code we will shortly be
663 // adjusting SP to allow some space for ESP. If we're returning to
664 // compiled code the saved sender SP was saved in sender_sp, so this
665 // restores it.
666 andr(sp, esp, -16);
667 }
668
669 #if INCLUDE_JFR
670 void InterpreterMacroAssembler::enter_jfr_critical_section() {
671 const Address sampling_critical_section(rthread, in_bytes(SAMPLING_CRITICAL_SECTION_OFFSET_JFR));
672 mov(rscratch1, true);
673 strb(rscratch1, sampling_critical_section);
712 Label slow_case;
713
714 // Load object pointer into obj_reg %c_rarg3
715 ldr(obj_reg, Address(lock_reg, obj_offset));
716
717 if (LockingMode == LM_LIGHTWEIGHT) {
718 lightweight_lock(lock_reg, obj_reg, tmp, tmp2, tmp3, slow_case);
719 b(done);
720 } else if (LockingMode == LM_LEGACY) {
721
722 if (DiagnoseSyncOnValueBasedClasses != 0) {
723 load_klass(tmp, obj_reg);
724 ldrb(tmp, Address(tmp, Klass::misc_flags_offset()));
725 tst(tmp, KlassFlags::_misc_is_value_based_class);
726 br(Assembler::NE, slow_case);
727 }
728
729 // Load (object->mark() | 1) into swap_reg
730 ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
731 orr(swap_reg, rscratch1, 1);
732
733 // Save (object->mark() | 1) into BasicLock's displaced header
734 str(swap_reg, Address(lock_reg, mark_offset));
735
736 assert(lock_offset == 0,
737 "displached header must be first word in BasicObjectLock");
738
739 Label fail;
740 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, count, /*fallthrough*/nullptr);
741
742 // Fast check for recursive lock.
743 //
744 // Can apply the optimization only if this is a stack lock
745 // allocated in this thread. For efficiency, we can focus on
746 // recently allocated stack locks (instead of reading the stack
747 // base and checking whether 'mark' points inside the current
748 // thread stack):
749 // 1) (mark & 7) == 0, and
750 // 2) sp <= mark < mark + os::pagesize()
751 //
1016 }
1017
1018
1019 void InterpreterMacroAssembler::profile_taken_branch(Register mdp) {
1020 if (ProfileInterpreter) {
1021 Label profile_continue;
1022
1023 // If no method data exists, go to profile_continue.
1024 test_method_data_pointer(mdp, profile_continue);
1025
1026 // We are taking a branch. Increment the taken count.
1027 increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
1028
1029 // The method data pointer needs to be updated to reflect the new target.
1030 update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
1031 bind(profile_continue);
1032 }
1033 }
1034
1035
1036 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
1037 if (ProfileInterpreter) {
1038 Label profile_continue;
1039
1040 // If no method data exists, go to profile_continue.
1041 test_method_data_pointer(mdp, profile_continue);
1042
1043 // We are not taking a branch. Increment the not taken count.
1044 increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
1045
1046 // The method data pointer needs to be updated to correspond to
1047 // the next bytecode
1048 update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));
1049 bind(profile_continue);
1050 }
1051 }
1052
1053
1054 void InterpreterMacroAssembler::profile_call(Register mdp) {
1055 if (ProfileInterpreter) {
1056 Label profile_continue;
1057
1058 // If no method data exists, go to profile_continue.
1059 test_method_data_pointer(mdp, profile_continue);
1060
1061 // We are making a call. Increment the count.
1062 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1063
1064 // The method data pointer needs to be updated to reflect the new target.
1065 update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1066 bind(profile_continue);
1067 }
1068 }
1351 // case_array_offset_in_bytes()
1352 movw(reg2, in_bytes(MultiBranchData::per_case_size()));
1353 movw(rscratch1, in_bytes(MultiBranchData::case_array_offset()));
1354 Assembler::maddw(index, index, reg2, rscratch1);
1355
1356 // Update the case count
1357 increment_mdp_data_at(mdp,
1358 index,
1359 in_bytes(MultiBranchData::relative_count_offset()));
1360
1361 // The method data pointer needs to be updated.
1362 update_mdp_by_offset(mdp,
1363 index,
1364 in_bytes(MultiBranchData::
1365 relative_displacement_offset()));
1366
1367 bind(profile_continue);
1368 }
1369 }
1370
1371 void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) {
1372 if (state == atos) {
1373 MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line);
1374 }
1375 }
1376
1377 void InterpreterMacroAssembler::notify_method_entry() {
1378 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1379 // track stack depth. If it is possible to enter interp_only_mode we add
1380 // the code to check if the event should be sent.
1381 if (JvmtiExport::can_post_interpreter_events()) {
1382 Label L;
1383 ldrw(r3, Address(rthread, JavaThread::interp_only_mode_offset()));
1384 cbzw(r3, L);
1385 call_VM(noreg, CAST_FROM_FN_PTR(address,
1386 InterpreterRuntime::post_method_entry));
1387 bind(L);
1388 }
1389
1390 if (DTraceMethodProbes) {
1649 profile_obj_type(tmp, mdo_arg_addr);
1650
1651 int to_add = in_bytes(TypeStackSlotEntries::per_arg_size());
1652 off_to_args += to_add;
1653 }
1654
1655 if (MethodData::profile_return()) {
1656 ldr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())));
1657 sub(tmp, tmp, TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count());
1658 }
1659
1660 add(rscratch1, mdp, off_to_args);
1661 bind(done);
1662 mov(mdp, rscratch1);
1663
1664 if (MethodData::profile_return()) {
1665 // We're right after the type profile for the last
1666 // argument. tmp is the number of cells left in the
1667 // CallTypeData/VirtualCallTypeData to reach its end. Non null
1668 // if there's a return to profile.
1669 assert(ReturnTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
1670 add(mdp, mdp, tmp, LSL, exact_log2(DataLayout::cell_size));
1671 }
1672 str(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1673 } else {
1674 assert(MethodData::profile_return(), "either profile call args or call ret");
1675 update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size()));
1676 }
1677
1678 // mdp points right after the end of the
1679 // CallTypeData/VirtualCallTypeData, right after the cells for the
1680 // return value type if there's one
1681
1682 bind(profile_continue);
1683 }
1684 }
1685
1686 void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) {
1687 assert_different_registers(mdp, ret, tmp, rbcp);
1688 if (ProfileInterpreter && MethodData::profile_return()) {
1689 Label profile_continue, done;
1695
1696 // If we don't profile all invoke bytecodes we must make sure
1697 // it's a bytecode we indeed profile. We can't go back to the
1698 // beginning of the ProfileData we intend to update to check its
1699 // type because we're right after it and we don't known its
1700 // length
1701 Label do_profile;
1702 ldrb(rscratch1, Address(rbcp, 0));
1703 cmp(rscratch1, (u1)Bytecodes::_invokedynamic);
1704 br(Assembler::EQ, do_profile);
1705 cmp(rscratch1, (u1)Bytecodes::_invokehandle);
1706 br(Assembler::EQ, do_profile);
1707 get_method(tmp);
1708 ldrh(rscratch1, Address(tmp, Method::intrinsic_id_offset()));
1709 subs(zr, rscratch1, static_cast<int>(vmIntrinsics::_compiledLambdaForm));
1710 br(Assembler::NE, profile_continue);
1711
1712 bind(do_profile);
1713 }
1714
1715 Address mdo_ret_addr(mdp, -in_bytes(ReturnTypeEntry::size()));
1716 mov(tmp, ret);
1717 profile_obj_type(tmp, mdo_ret_addr);
1718
1719 bind(profile_continue);
1720 }
1721 }
1722
1723 void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2) {
1724 assert_different_registers(rscratch1, rscratch2, mdp, tmp1, tmp2);
1725 if (ProfileInterpreter && MethodData::profile_parameters()) {
1726 Label profile_continue, done;
1727
1728 test_method_data_pointer(mdp, profile_continue);
1729
1730 // Load the offset of the area within the MDO used for
1731 // parameters. If it's negative we're not profiling any parameters
1732 ldrw(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset())));
1733 tbnz(tmp1, 31, profile_continue); // i.e. sign bit set
1734
1735 // Compute a pointer to the area for parameters from the offset
|
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "compiler/compiler_globals.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/barrierSetAssembler.hpp"
30 #include "interp_masm_aarch64.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interpreterRuntime.hpp"
33 #include "logging/log.hpp"
34 #include "oops/arrayOop.hpp"
35 #include "oops/constMethodFlags.hpp"
36 #include "oops/markWord.hpp"
37 #include "oops/method.hpp"
38 #include "oops/methodData.hpp"
39 #include "oops/inlineKlass.hpp"
40 #include "oops/resolvedFieldEntry.hpp"
41 #include "oops/resolvedIndyEntry.hpp"
42 #include "oops/resolvedMethodEntry.hpp"
43 #include "prims/jvmtiExport.hpp"
44 #include "prims/jvmtiThreadState.hpp"
45 #include "runtime/basicLock.hpp"
46 #include "runtime/frame.inline.hpp"
47 #include "runtime/javaThread.hpp"
48 #include "runtime/safepointMechanism.hpp"
49 #include "runtime/sharedRuntime.hpp"
50 #include "utilities/powerOfTwo.hpp"
51
52 void InterpreterMacroAssembler::narrow(Register result) {
53
54 // Get method->_constMethod->_result_type
55 ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
56 ldr(rscratch1, Address(rscratch1, Method::const_offset()));
57 ldrb(rscratch1, Address(rscratch1, ConstMethod::result_type_offset()));
58
59 Label done, notBool, notByte, notChar;
193 ldrw(index, Address(rbcp, bcp_offset));
194 } else if (index_size == sizeof(u1)) {
195 load_unsigned_byte(index, Address(rbcp, bcp_offset));
196 } else {
197 ShouldNotReachHere();
198 }
199 }
200
201 void InterpreterMacroAssembler::get_method_counters(Register method,
202 Register mcs, Label& skip) {
203 Label has_counters;
204 ldr(mcs, Address(method, Method::method_counters_offset()));
205 cbnz(mcs, has_counters);
206 call_VM(noreg, CAST_FROM_FN_PTR(address,
207 InterpreterRuntime::build_method_counters), method);
208 ldr(mcs, Address(method, Method::method_counters_offset()));
209 cbz(mcs, skip); // No MethodCounters allocated, OutOfMemory
210 bind(has_counters);
211 }
212
213 void InterpreterMacroAssembler::allocate_instance(Register klass, Register new_obj,
214 Register t1, Register t2,
215 bool clear_fields, Label& alloc_failed) {
216 MacroAssembler::allocate_instance(klass, new_obj, t1, t2, clear_fields, alloc_failed);
217 if (DTraceAllocProbes) {
218 // Trigger dtrace event for fastpath
219 push(atos);
220 call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), new_obj);
221 pop(atos);
222 }
223 }
224
225 void InterpreterMacroAssembler::read_flat_field(Register entry,
226 Register field_index, Register field_offset,
227 Register temp, Register obj) {
228 Label failed_alloc, slow_path, done;
229 const Register src = field_offset;
230 const Register alloc_temp = r10;
231 const Register dst_temp = field_index;
232 const Register layout_info = temp;
233 assert_different_registers(obj, entry, field_index, field_offset, temp, alloc_temp, rscratch1);
234
235 load_unsigned_byte(temp, Address(entry, in_bytes(ResolvedFieldEntry::flags_offset())));
236 // If the field is nullable, jump to slow path
237 tbz(temp, ResolvedFieldEntry::is_null_free_inline_type_shift, slow_path);
238
239 // Grab the inline field klass
240 ldr(rscratch1, Address(entry, in_bytes(ResolvedFieldEntry::field_holder_offset())));
241 inline_layout_info(rscratch1, field_index, layout_info);
242
243 const Register field_klass = dst_temp;
244 ldr(field_klass, Address(layout_info, in_bytes(InlineLayoutInfo::klass_offset())));
245
246 // allocate buffer
247 push(obj); // save holder
248 allocate_instance(field_klass, obj, alloc_temp, rscratch2, false, failed_alloc);
249
250 // Have an oop instance buffer, copy into it
251 payload_address(obj, dst_temp, field_klass); // danger, uses rscratch1
252 pop(alloc_temp); // restore holder
253 lea(src, Address(alloc_temp, field_offset));
254 // call_VM_leaf, clobbers a few regs, save restore new obj
255 push(obj);
256 flat_field_copy(IS_DEST_UNINITIALIZED, src, dst_temp, layout_info);
257 pop(obj);
258 b(done);
259
260 bind(failed_alloc);
261 pop(obj);
262 bind(slow_path);
263 call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flat_field),
264 obj, entry);
265
266 bind(done);
267 membar(Assembler::StoreStore);
268 }
269
270 void InterpreterMacroAssembler::write_flat_field(Register entry, Register field_offset,
271 Register tmp1, Register tmp2,
272 Register obj) {
273 assert_different_registers(entry, field_offset, tmp1, tmp2, obj);
274 Label slow_path, done;
275
276 load_unsigned_byte(tmp1, Address(entry, in_bytes(ResolvedFieldEntry::flags_offset())));
277 test_field_is_not_null_free_inline_type(tmp1, noreg /* temp */, slow_path);
278
279 null_check(r0); // FIXME JDK-8341120
280
281 add(obj, obj, field_offset);
282
283 load_klass(tmp1, r0);
284 payload_address(r0, r0, tmp1);
285
286 Register layout_info = field_offset;
287 load_unsigned_short(tmp1, Address(entry, in_bytes(ResolvedFieldEntry::field_index_offset())));
288 ldr(tmp2, Address(entry, in_bytes(ResolvedFieldEntry::field_holder_offset())));
289 inline_layout_info(tmp2, tmp1, layout_info);
290
291 flat_field_copy(IN_HEAP, r0, obj, layout_info);
292 b(done);
293
294 bind(slow_path);
295 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_flat_field), obj, r0, entry);
296 bind(done);
297 }
298
299 // Load object from cpool->resolved_references(index)
300 void InterpreterMacroAssembler::load_resolved_reference_at_index(
301 Register result, Register index, Register tmp) {
302 assert_different_registers(result, index);
303
304 get_constant_pool(result);
305 // load pointer for resolved_references[] objArray
306 ldr(result, Address(result, ConstantPool::cache_offset()));
307 ldr(result, Address(result, ConstantPoolCache::resolved_references_offset()));
308 resolve_oop_handle(result, tmp, rscratch2);
309 // Add in the index
310 add(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
311 load_heap_oop(result, Address(result, index, Address::uxtw(LogBytesPerHeapOop)), tmp, rscratch2);
312 }
313
314 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
315 Register cpool, Register index, Register klass, Register temp) {
316 add(temp, cpool, index, LSL, LogBytesPerWord);
317 ldrh(temp, Address(temp, sizeof(ConstantPool))); // temp = resolved_klass_index
318 ldr(klass, Address(cpool, ConstantPool::resolved_klasses_offset())); // klass = cpool->_resolved_klasses
319 add(klass, klass, temp, LSL, LogBytesPerWord);
320 ldr(klass, Address(klass, Array<Klass*>::base_offset_in_bytes()));
321 }
322
323 // Generate a subtype check: branch to ok_is_subtype if sub_klass is a
324 // subtype of super_klass.
325 //
326 // Args:
327 // r0: superklass
328 // Rsub_klass: subklass
329 //
330 // Kills:
331 // r2, r5
332 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
333 Label& ok_is_subtype,
334 bool profile) {
335 assert(Rsub_klass != r0, "r0 holds superklass");
336 assert(Rsub_klass != r2, "r2 holds 2ndary super array length");
337 assert(Rsub_klass != r5, "r5 holds 2ndary super array scan ptr");
338
339 // Profile the not-null value's klass.
340 if (profile) {
341 profile_typecheck(r2, Rsub_klass, r5); // blows r2, reloads r5
342 }
343
344 // Do the check.
345 check_klass_subtype(Rsub_klass, r0, r2, ok_is_subtype); // blows r2
346 }
347
348 // Java Expression Stack
349
350 void InterpreterMacroAssembler::pop_ptr(Register r) {
351 ldr(r, post(esp, wordSize));
352 }
353
354 void InterpreterMacroAssembler::pop_i(Register r) {
355 ldrw(r, post(esp, wordSize));
356 }
357
358 void InterpreterMacroAssembler::pop_l(Register r) {
359 ldr(r, post(esp, 2 * Interpreter::stackElementSize));
360 }
361
362 void InterpreterMacroAssembler::push_ptr(Register r) {
698 br(Assembler::AL, fast_path);
699 bind(slow_path);
700 push(state);
701 set_last_Java_frame(esp, rfp, pc(), rscratch1);
702 super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), rthread);
703 reset_last_Java_frame(true);
704 pop(state);
705 bind(fast_path);
706
707 // JVMTI support. Make sure the safepoint poll test is issued prior.
708 if (notify_jvmdi) {
709 notify_method_exit(state, NotifyJVMTI); // preserve TOSCA
710 } else {
711 notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
712 }
713
714 // remove activation
715 // get sender esp
716 ldr(rscratch2,
717 Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
718
719 if (StackReservedPages > 0) {
720 // testing if reserved zone needs to be re-enabled
721 Label no_reserved_zone_enabling;
722
723 // check if already enabled - if so no re-enabling needed
724 assert(sizeof(StackOverflow::StackGuardState) == 4, "unexpected size");
725 ldrw(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
726 cmpw(rscratch1, (u1)StackOverflow::stack_guard_enabled);
727 br(Assembler::EQ, no_reserved_zone_enabling);
728
729 // look for an overflow into the stack reserved zone, i.e.
730 // interpreter_frame_sender_sp <= JavaThread::reserved_stack_activation
731 ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset()));
732 cmp(rscratch2, rscratch1);
733 br(Assembler::LS, no_reserved_zone_enabling);
734
735 JFR_ONLY(leave_jfr_critical_section();)
736
737 call_VM_leaf(
738 CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), rthread);
739 call_VM(noreg, CAST_FROM_FN_PTR(address,
740 InterpreterRuntime::throw_delayed_StackOverflowError));
741 should_not_reach_here();
742
743 bind(no_reserved_zone_enabling);
744 }
745
746 if (state == atos && InlineTypeReturnedAsFields) {
747 Label skip;
748 Label not_null;
749 cbnz(r0, not_null);
750 // Returned value is null, zero all return registers because they may belong to oop fields
751 mov(j_rarg1, zr);
752 mov(j_rarg2, zr);
753 mov(j_rarg3, zr);
754 mov(j_rarg4, zr);
755 mov(j_rarg5, zr);
756 mov(j_rarg6, zr);
757 mov(j_rarg7, zr);
758 b(skip);
759 bind(not_null);
760
761 // Check if we are returning an non-null inline type and load its fields into registers
762 test_oop_is_not_inline_type(r0, rscratch2, skip, /* can_be_null= */ false);
763
764 // Load fields from a buffered value with an inline class specific handler
765 load_klass(rscratch1 /*dst*/, r0 /*src*/);
766 ldr(rscratch1, Address(rscratch1, InstanceKlass::adr_inlineklass_fixed_block_offset()));
767 ldr(rscratch1, Address(rscratch1, InlineKlass::unpack_handler_offset()));
768 // Unpack handler can be null if inline type is not scalarizable in returns
769 cbz(rscratch1, skip);
770
771 blr(rscratch1);
772 #ifdef ASSERT
773 // TODO 8284443 Enable
774 if (StressCallingConvention && false) {
775 Label skip_stress;
776 ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
777 ldrw(rscratch1, Address(rscratch1, Method::flags_offset()));
778 tstw(rscratch1, MethodFlags::has_scalarized_return_flag());
779 br(Assembler::EQ, skip_stress);
780 load_klass(r0, r0);
781 orr(r0, r0, 1);
782 bind(skip_stress);
783 }
784 #endif
785 bind(skip);
786 // Check above kills sender esp in rscratch2. Reload it.
787 ldr(rscratch2, Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
788 }
789
790 // remove frame anchor
791 leave();
792
793 JFR_ONLY(leave_jfr_critical_section();)
794
795 // restore sender esp
796 mov(esp, rscratch2);
797
798 // If we're returning to interpreted code we will shortly be
799 // adjusting SP to allow some space for ESP. If we're returning to
800 // compiled code the saved sender SP was saved in sender_sp, so this
801 // restores it.
802 andr(sp, esp, -16);
803 }
804
805 #if INCLUDE_JFR
806 void InterpreterMacroAssembler::enter_jfr_critical_section() {
807 const Address sampling_critical_section(rthread, in_bytes(SAMPLING_CRITICAL_SECTION_OFFSET_JFR));
808 mov(rscratch1, true);
809 strb(rscratch1, sampling_critical_section);
848 Label slow_case;
849
850 // Load object pointer into obj_reg %c_rarg3
851 ldr(obj_reg, Address(lock_reg, obj_offset));
852
853 if (LockingMode == LM_LIGHTWEIGHT) {
854 lightweight_lock(lock_reg, obj_reg, tmp, tmp2, tmp3, slow_case);
855 b(done);
856 } else if (LockingMode == LM_LEGACY) {
857
858 if (DiagnoseSyncOnValueBasedClasses != 0) {
859 load_klass(tmp, obj_reg);
860 ldrb(tmp, Address(tmp, Klass::misc_flags_offset()));
861 tst(tmp, KlassFlags::_misc_is_value_based_class);
862 br(Assembler::NE, slow_case);
863 }
864
865 // Load (object->mark() | 1) into swap_reg
866 ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
867 orr(swap_reg, rscratch1, 1);
868 if (EnableValhalla) {
869 // Mask inline_type bit such that we go to the slow path if object is an inline type
870 andr(swap_reg, swap_reg, ~((int) markWord::inline_type_bit_in_place));
871 }
872
873 // Save (object->mark() | 1) into BasicLock's displaced header
874 str(swap_reg, Address(lock_reg, mark_offset));
875
876 assert(lock_offset == 0,
877 "displached header must be first word in BasicObjectLock");
878
879 Label fail;
880 cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, rscratch1, count, /*fallthrough*/nullptr);
881
882 // Fast check for recursive lock.
883 //
884 // Can apply the optimization only if this is a stack lock
885 // allocated in this thread. For efficiency, we can focus on
886 // recently allocated stack locks (instead of reading the stack
887 // base and checking whether 'mark' points inside the current
888 // thread stack):
889 // 1) (mark & 7) == 0, and
890 // 2) sp <= mark < mark + os::pagesize()
891 //
1156 }
1157
1158
1159 void InterpreterMacroAssembler::profile_taken_branch(Register mdp) {
1160 if (ProfileInterpreter) {
1161 Label profile_continue;
1162
1163 // If no method data exists, go to profile_continue.
1164 test_method_data_pointer(mdp, profile_continue);
1165
1166 // We are taking a branch. Increment the taken count.
1167 increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
1168
1169 // The method data pointer needs to be updated to reflect the new target.
1170 update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
1171 bind(profile_continue);
1172 }
1173 }
1174
1175
1176 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp, bool acmp) {
1177 if (ProfileInterpreter) {
1178 Label profile_continue;
1179
1180 // If no method data exists, go to profile_continue.
1181 test_method_data_pointer(mdp, profile_continue);
1182
1183 // We are not taking a branch. Increment the not taken count.
1184 increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
1185
1186 // The method data pointer needs to be updated to correspond to
1187 // the next bytecode
1188 update_mdp_by_constant(mdp, acmp ? in_bytes(ACmpData::acmp_data_size()) : in_bytes(BranchData::branch_data_size()));
1189 bind(profile_continue);
1190 }
1191 }
1192
1193
1194 void InterpreterMacroAssembler::profile_call(Register mdp) {
1195 if (ProfileInterpreter) {
1196 Label profile_continue;
1197
1198 // If no method data exists, go to profile_continue.
1199 test_method_data_pointer(mdp, profile_continue);
1200
1201 // We are making a call. Increment the count.
1202 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1203
1204 // The method data pointer needs to be updated to reflect the new target.
1205 update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1206 bind(profile_continue);
1207 }
1208 }
1491 // case_array_offset_in_bytes()
1492 movw(reg2, in_bytes(MultiBranchData::per_case_size()));
1493 movw(rscratch1, in_bytes(MultiBranchData::case_array_offset()));
1494 Assembler::maddw(index, index, reg2, rscratch1);
1495
1496 // Update the case count
1497 increment_mdp_data_at(mdp,
1498 index,
1499 in_bytes(MultiBranchData::relative_count_offset()));
1500
1501 // The method data pointer needs to be updated.
1502 update_mdp_by_offset(mdp,
1503 index,
1504 in_bytes(MultiBranchData::
1505 relative_displacement_offset()));
1506
1507 bind(profile_continue);
1508 }
1509 }
1510
1511 template <class ArrayData> void InterpreterMacroAssembler::profile_array_type(Register mdp,
1512 Register array,
1513 Register tmp) {
1514 if (ProfileInterpreter) {
1515 Label profile_continue;
1516
1517 // If no method data exists, go to profile_continue.
1518 test_method_data_pointer(mdp, profile_continue);
1519
1520 mov(tmp, array);
1521 profile_obj_type(tmp, Address(mdp, in_bytes(ArrayData::array_offset())));
1522
1523 Label not_flat;
1524 test_non_flat_array_oop(array, tmp, not_flat);
1525
1526 set_mdp_flag_at(mdp, ArrayData::flat_array_byte_constant());
1527
1528 bind(not_flat);
1529
1530 Label not_null_free;
1531 test_non_null_free_array_oop(array, tmp, not_null_free);
1532
1533 set_mdp_flag_at(mdp, ArrayData::null_free_array_byte_constant());
1534
1535 bind(not_null_free);
1536
1537 bind(profile_continue);
1538 }
1539 }
1540
1541 template void InterpreterMacroAssembler::profile_array_type<ArrayLoadData>(Register mdp,
1542 Register array,
1543 Register tmp);
1544 template void InterpreterMacroAssembler::profile_array_type<ArrayStoreData>(Register mdp,
1545 Register array,
1546 Register tmp);
1547
1548 void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2) {
1549 if (ProfileInterpreter) {
1550 Label profile_continue;
1551
1552 // If no method data exists, go to profile_continue.
1553 test_method_data_pointer(mdp, profile_continue);
1554
1555 Label done, update;
1556 cbnz(element, update);
1557 set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
1558 b(done);
1559
1560 bind(update);
1561 load_klass(tmp, element);
1562
1563 // Record the object type.
1564 record_klass_in_profile(tmp, mdp, tmp2);
1565
1566 bind(done);
1567
1568 // The method data pointer needs to be updated.
1569 update_mdp_by_constant(mdp, in_bytes(ArrayStoreData::array_store_data_size()));
1570
1571 bind(profile_continue);
1572 }
1573 }
1574
1575
1576 void InterpreterMacroAssembler::profile_element_type(Register mdp,
1577 Register element,
1578 Register tmp) {
1579 if (ProfileInterpreter) {
1580 Label profile_continue;
1581
1582 // If no method data exists, go to profile_continue.
1583 test_method_data_pointer(mdp, profile_continue);
1584
1585 mov(tmp, element);
1586 profile_obj_type(tmp, Address(mdp, in_bytes(ArrayLoadData::element_offset())));
1587
1588 // The method data pointer needs to be updated.
1589 update_mdp_by_constant(mdp, in_bytes(ArrayLoadData::array_load_data_size()));
1590
1591 bind(profile_continue);
1592 }
1593 }
1594
1595 void InterpreterMacroAssembler::profile_acmp(Register mdp,
1596 Register left,
1597 Register right,
1598 Register tmp) {
1599 if (ProfileInterpreter) {
1600 Label profile_continue;
1601
1602 // If no method data exists, go to profile_continue.
1603 test_method_data_pointer(mdp, profile_continue);
1604
1605 mov(tmp, left);
1606 profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::left_offset())));
1607
1608 Label left_not_inline_type;
1609 test_oop_is_not_inline_type(left, tmp, left_not_inline_type);
1610 set_mdp_flag_at(mdp, ACmpData::left_inline_type_byte_constant());
1611 bind(left_not_inline_type);
1612
1613 mov(tmp, right);
1614 profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::right_offset())));
1615
1616 Label right_not_inline_type;
1617 test_oop_is_not_inline_type(right, tmp, right_not_inline_type);
1618 set_mdp_flag_at(mdp, ACmpData::right_inline_type_byte_constant());
1619 bind(right_not_inline_type);
1620
1621 bind(profile_continue);
1622 }
1623 }
1624
1625 void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) {
1626 if (state == atos) {
1627 MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line);
1628 }
1629 }
1630
1631 void InterpreterMacroAssembler::notify_method_entry() {
1632 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1633 // track stack depth. If it is possible to enter interp_only_mode we add
1634 // the code to check if the event should be sent.
1635 if (JvmtiExport::can_post_interpreter_events()) {
1636 Label L;
1637 ldrw(r3, Address(rthread, JavaThread::interp_only_mode_offset()));
1638 cbzw(r3, L);
1639 call_VM(noreg, CAST_FROM_FN_PTR(address,
1640 InterpreterRuntime::post_method_entry));
1641 bind(L);
1642 }
1643
1644 if (DTraceMethodProbes) {
1903 profile_obj_type(tmp, mdo_arg_addr);
1904
1905 int to_add = in_bytes(TypeStackSlotEntries::per_arg_size());
1906 off_to_args += to_add;
1907 }
1908
1909 if (MethodData::profile_return()) {
1910 ldr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())));
1911 sub(tmp, tmp, TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count());
1912 }
1913
1914 add(rscratch1, mdp, off_to_args);
1915 bind(done);
1916 mov(mdp, rscratch1);
1917
1918 if (MethodData::profile_return()) {
1919 // We're right after the type profile for the last
1920 // argument. tmp is the number of cells left in the
1921 // CallTypeData/VirtualCallTypeData to reach its end. Non null
1922 // if there's a return to profile.
1923 assert(SingleTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
1924 add(mdp, mdp, tmp, LSL, exact_log2(DataLayout::cell_size));
1925 }
1926 str(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1927 } else {
1928 assert(MethodData::profile_return(), "either profile call args or call ret");
1929 update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size()));
1930 }
1931
1932 // mdp points right after the end of the
1933 // CallTypeData/VirtualCallTypeData, right after the cells for the
1934 // return value type if there's one
1935
1936 bind(profile_continue);
1937 }
1938 }
1939
1940 void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) {
1941 assert_different_registers(mdp, ret, tmp, rbcp);
1942 if (ProfileInterpreter && MethodData::profile_return()) {
1943 Label profile_continue, done;
1949
1950 // If we don't profile all invoke bytecodes we must make sure
1951 // it's a bytecode we indeed profile. We can't go back to the
1952 // beginning of the ProfileData we intend to update to check its
1953 // type because we're right after it and we don't known its
1954 // length
1955 Label do_profile;
1956 ldrb(rscratch1, Address(rbcp, 0));
1957 cmp(rscratch1, (u1)Bytecodes::_invokedynamic);
1958 br(Assembler::EQ, do_profile);
1959 cmp(rscratch1, (u1)Bytecodes::_invokehandle);
1960 br(Assembler::EQ, do_profile);
1961 get_method(tmp);
1962 ldrh(rscratch1, Address(tmp, Method::intrinsic_id_offset()));
1963 subs(zr, rscratch1, static_cast<int>(vmIntrinsics::_compiledLambdaForm));
1964 br(Assembler::NE, profile_continue);
1965
1966 bind(do_profile);
1967 }
1968
1969 Address mdo_ret_addr(mdp, -in_bytes(SingleTypeEntry::size()));
1970 mov(tmp, ret);
1971 profile_obj_type(tmp, mdo_ret_addr);
1972
1973 bind(profile_continue);
1974 }
1975 }
1976
1977 void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2) {
1978 assert_different_registers(rscratch1, rscratch2, mdp, tmp1, tmp2);
1979 if (ProfileInterpreter && MethodData::profile_parameters()) {
1980 Label profile_continue, done;
1981
1982 test_method_data_pointer(mdp, profile_continue);
1983
1984 // Load the offset of the area within the MDO used for
1985 // parameters. If it's negative we're not profiling any parameters
1986 ldrw(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset())));
1987 tbnz(tmp1, 31, profile_continue); // i.e. sign bit set
1988
1989 // Compute a pointer to the area for parameters from the offset
|