15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "compiler/compiler_globals.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/barrierSetAssembler.hpp"
30 #include "interp_masm_aarch64.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interpreterRuntime.hpp"
33 #include "logging/log.hpp"
34 #include "oops/arrayOop.hpp"
35 #include "oops/markWord.hpp"
36 #include "oops/method.hpp"
37 #include "oops/methodData.hpp"
38 #include "oops/resolvedFieldEntry.hpp"
39 #include "oops/resolvedIndyEntry.hpp"
40 #include "oops/resolvedMethodEntry.hpp"
41 #include "prims/jvmtiExport.hpp"
42 #include "prims/jvmtiThreadState.hpp"
43 #include "runtime/basicLock.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/javaThread.hpp"
46 #include "runtime/safepointMechanism.hpp"
47 #include "runtime/sharedRuntime.hpp"
48 #include "utilities/powerOfTwo.hpp"
49
50 void InterpreterMacroAssembler::narrow(Register result) {
51
52 // Get method->_constMethod->_result_type
53 ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
54 ldr(rscratch1, Address(rscratch1, Method::const_offset()));
55 ldrb(rscratch1, Address(rscratch1, ConstMethod::result_type_offset()));
56
57 Label done, notBool, notByte, notChar;
191 ldrw(index, Address(rbcp, bcp_offset));
192 } else if (index_size == sizeof(u1)) {
193 load_unsigned_byte(index, Address(rbcp, bcp_offset));
194 } else {
195 ShouldNotReachHere();
196 }
197 }
198
199 void InterpreterMacroAssembler::get_method_counters(Register method,
200 Register mcs, Label& skip) {
201 Label has_counters;
202 ldr(mcs, Address(method, Method::method_counters_offset()));
203 cbnz(mcs, has_counters);
204 call_VM(noreg, CAST_FROM_FN_PTR(address,
205 InterpreterRuntime::build_method_counters), method);
206 ldr(mcs, Address(method, Method::method_counters_offset()));
207 cbz(mcs, skip); // No MethodCounters allocated, OutOfMemory
208 bind(has_counters);
209 }
210
211 // Load object from cpool->resolved_references(index)
212 void InterpreterMacroAssembler::load_resolved_reference_at_index(
213 Register result, Register index, Register tmp) {
214 assert_different_registers(result, index);
215
216 get_constant_pool(result);
217 // load pointer for resolved_references[] objArray
218 ldr(result, Address(result, ConstantPool::cache_offset()));
219 ldr(result, Address(result, ConstantPoolCache::resolved_references_offset()));
220 resolve_oop_handle(result, tmp, rscratch2);
221 // Add in the index
222 add(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
223 load_heap_oop(result, Address(result, index, Address::uxtw(LogBytesPerHeapOop)), tmp, rscratch2);
224 }
225
226 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
227 Register cpool, Register index, Register klass, Register temp) {
228 add(temp, cpool, index, LSL, LogBytesPerWord);
229 ldrh(temp, Address(temp, sizeof(ConstantPool))); // temp = resolved_klass_index
230 ldr(klass, Address(cpool, ConstantPool::resolved_klasses_offset())); // klass = cpool->_resolved_klasses
231 add(klass, klass, temp, LSL, LogBytesPerWord);
232 ldr(klass, Address(klass, Array<Klass*>::base_offset_in_bytes()));
233 }
234
235 // Generate a subtype check: branch to ok_is_subtype if sub_klass is a
236 // subtype of super_klass.
237 //
238 // Args:
239 // r0: superklass
240 // Rsub_klass: subklass
241 //
242 // Kills:
243 // r2
244 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
245 Label& ok_is_subtype) {
246 assert(Rsub_klass != r0, "r0 holds superklass");
247 assert(Rsub_klass != r2, "r2 holds 2ndary super array length");
248
249 // Profile the not-null value's klass.
250 profile_typecheck(r2, Rsub_klass); // blows r2
251
252 // Do the check.
253 check_klass_subtype(Rsub_klass, r0, r2, ok_is_subtype); // blows r2
254 }
255
256 // Java Expression Stack
257
258 void InterpreterMacroAssembler::pop_ptr(Register r) {
259 ldr(r, post(esp, wordSize));
260 }
261
262 void InterpreterMacroAssembler::pop_i(Register r) {
263 ldrw(r, post(esp, wordSize));
264 }
265
266 void InterpreterMacroAssembler::pop_l(Register r) {
267 ldr(r, post(esp, 2 * Interpreter::stackElementSize));
268 }
269
270 void InterpreterMacroAssembler::push_ptr(Register r) {
271 str(r, pre(esp, -wordSize));
641 cmpw(rscratch1, (u1)StackOverflow::stack_guard_enabled);
642 br(Assembler::EQ, no_reserved_zone_enabling);
643
644 // look for an overflow into the stack reserved zone, i.e.
645 // interpreter_frame_sender_sp <= JavaThread::reserved_stack_activation
646 ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset()));
647 cmp(rscratch2, rscratch1);
648 br(Assembler::LS, no_reserved_zone_enabling);
649
650 JFR_ONLY(leave_jfr_critical_section();)
651
652 call_VM_leaf(
653 CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), rthread);
654 call_VM(noreg, CAST_FROM_FN_PTR(address,
655 InterpreterRuntime::throw_delayed_StackOverflowError));
656 should_not_reach_here();
657
658 bind(no_reserved_zone_enabling);
659 }
660
661 // remove frame anchor
662 leave();
663
664 JFR_ONLY(leave_jfr_critical_section();)
665
666 // restore sender esp
667 mov(esp, rscratch2);
668
669 // If we're returning to interpreted code we will shortly be
670 // adjusting SP to allow some space for ESP. If we're returning to
671 // compiled code the saved sender SP was saved in sender_sp, so this
672 // restores it.
673 andr(sp, esp, -16);
674 }
675
676 #if INCLUDE_JFR
677 void InterpreterMacroAssembler::enter_jfr_critical_section() {
678 const Address sampling_critical_section(rthread, in_bytes(SAMPLING_CRITICAL_SECTION_OFFSET_JFR));
679 mov(rscratch1, true);
680 strb(rscratch1, sampling_critical_section);
918 }
919
920
921 void InterpreterMacroAssembler::profile_taken_branch(Register mdp) {
922 if (ProfileInterpreter) {
923 Label profile_continue;
924
925 // If no method data exists, go to profile_continue.
926 test_method_data_pointer(mdp, profile_continue);
927
928 // We are taking a branch. Increment the taken count.
929 increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
930
931 // The method data pointer needs to be updated to reflect the new target.
932 update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
933 bind(profile_continue);
934 }
935 }
936
937
938 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
939 if (ProfileInterpreter) {
940 Label profile_continue;
941
942 // If no method data exists, go to profile_continue.
943 test_method_data_pointer(mdp, profile_continue);
944
945 // We are not taking a branch. Increment the not taken count.
946 increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
947
948 // The method data pointer needs to be updated to correspond to
949 // the next bytecode
950 update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));
951 bind(profile_continue);
952 }
953 }
954
955
956 void InterpreterMacroAssembler::profile_call(Register mdp) {
957 if (ProfileInterpreter) {
958 Label profile_continue;
959
960 // If no method data exists, go to profile_continue.
961 test_method_data_pointer(mdp, profile_continue);
962
963 // We are making a call. Increment the count.
964 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
965
966 // The method data pointer needs to be updated to reflect the new target.
967 update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
968 bind(profile_continue);
969 }
970 }
1116 // case_array_offset_in_bytes()
1117 movw(reg2, in_bytes(MultiBranchData::per_case_size()));
1118 movw(rscratch1, in_bytes(MultiBranchData::case_array_offset()));
1119 Assembler::maddw(index, index, reg2, rscratch1);
1120
1121 // Update the case count
1122 increment_mdp_data_at(mdp,
1123 index,
1124 in_bytes(MultiBranchData::relative_count_offset()));
1125
1126 // The method data pointer needs to be updated.
1127 update_mdp_by_offset(mdp,
1128 index,
1129 in_bytes(MultiBranchData::
1130 relative_displacement_offset()));
1131
1132 bind(profile_continue);
1133 }
1134 }
1135
1136 void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) {
1137 if (state == atos) {
1138 MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line);
1139 }
1140 }
1141
1142 void InterpreterMacroAssembler::notify_method_entry() {
1143 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1144 // track stack depth. If it is possible to enter interp_only_mode we add
1145 // the code to check if the event should be sent.
1146 if (JvmtiExport::can_post_interpreter_events()) {
1147 Label L;
1148 ldrw(r3, Address(rthread, JavaThread::interp_only_mode_offset()));
1149 cbzw(r3, L);
1150 call_VM(noreg, CAST_FROM_FN_PTR(address,
1151 InterpreterRuntime::post_method_entry));
1152 bind(L);
1153 }
1154
1155 if (DTraceMethodProbes) {
1479 profile_obj_type(tmp, mdo_arg_addr);
1480
1481 int to_add = in_bytes(TypeStackSlotEntries::per_arg_size());
1482 off_to_args += to_add;
1483 }
1484
1485 if (MethodData::profile_return()) {
1486 ldr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())));
1487 sub(tmp, tmp, TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count());
1488 }
1489
1490 add(rscratch1, mdp, off_to_args);
1491 bind(done);
1492 mov(mdp, rscratch1);
1493
1494 if (MethodData::profile_return()) {
1495 // We're right after the type profile for the last
1496 // argument. tmp is the number of cells left in the
1497 // CallTypeData/VirtualCallTypeData to reach its end. Non null
1498 // if there's a return to profile.
1499 assert(ReturnTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
1500 add(mdp, mdp, tmp, LSL, exact_log2(DataLayout::cell_size));
1501 }
1502 str(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1503 } else {
1504 assert(MethodData::profile_return(), "either profile call args or call ret");
1505 update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size()));
1506 }
1507
1508 // mdp points right after the end of the
1509 // CallTypeData/VirtualCallTypeData, right after the cells for the
1510 // return value type if there's one
1511
1512 bind(profile_continue);
1513 }
1514 }
1515
1516 void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) {
1517 assert_different_registers(mdp, ret, tmp, rbcp);
1518 if (ProfileInterpreter && MethodData::profile_return()) {
1519 Label profile_continue, done;
1525
1526 // If we don't profile all invoke bytecodes we must make sure
1527 // it's a bytecode we indeed profile. We can't go back to the
1528 // beginning of the ProfileData we intend to update to check its
1529 // type because we're right after it and we don't known its
1530 // length
1531 Label do_profile;
1532 ldrb(rscratch1, Address(rbcp, 0));
1533 cmp(rscratch1, (u1)Bytecodes::_invokedynamic);
1534 br(Assembler::EQ, do_profile);
1535 cmp(rscratch1, (u1)Bytecodes::_invokehandle);
1536 br(Assembler::EQ, do_profile);
1537 get_method(tmp);
1538 ldrh(rscratch1, Address(tmp, Method::intrinsic_id_offset()));
1539 subs(zr, rscratch1, static_cast<int>(vmIntrinsics::_compiledLambdaForm));
1540 br(Assembler::NE, profile_continue);
1541
1542 bind(do_profile);
1543 }
1544
1545 Address mdo_ret_addr(mdp, -in_bytes(ReturnTypeEntry::size()));
1546 mov(tmp, ret);
1547 profile_obj_type(tmp, mdo_ret_addr);
1548
1549 bind(profile_continue);
1550 }
1551 }
1552
1553 void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2) {
1554 assert_different_registers(rscratch1, rscratch2, mdp, tmp1, tmp2);
1555 if (ProfileInterpreter && MethodData::profile_parameters()) {
1556 Label profile_continue, done;
1557
1558 test_method_data_pointer(mdp, profile_continue);
1559
1560 // Load the offset of the area within the MDO used for
1561 // parameters. If it's negative we're not profiling any parameters
1562 ldrw(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset())));
1563 tbnz(tmp1, 31, profile_continue); // i.e. sign bit set
1564
1565 // Compute a pointer to the area for parameters from the offset
|
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "compiler/compiler_globals.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/barrierSetAssembler.hpp"
30 #include "interp_masm_aarch64.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interpreterRuntime.hpp"
33 #include "logging/log.hpp"
34 #include "oops/arrayOop.hpp"
35 #include "oops/constMethodFlags.hpp"
36 #include "oops/markWord.hpp"
37 #include "oops/method.hpp"
38 #include "oops/methodData.hpp"
39 #include "oops/inlineKlass.hpp"
40 #include "oops/resolvedFieldEntry.hpp"
41 #include "oops/resolvedIndyEntry.hpp"
42 #include "oops/resolvedMethodEntry.hpp"
43 #include "prims/jvmtiExport.hpp"
44 #include "prims/jvmtiThreadState.hpp"
45 #include "runtime/basicLock.hpp"
46 #include "runtime/frame.inline.hpp"
47 #include "runtime/javaThread.hpp"
48 #include "runtime/safepointMechanism.hpp"
49 #include "runtime/sharedRuntime.hpp"
50 #include "utilities/powerOfTwo.hpp"
51
52 void InterpreterMacroAssembler::narrow(Register result) {
53
54 // Get method->_constMethod->_result_type
55 ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
56 ldr(rscratch1, Address(rscratch1, Method::const_offset()));
57 ldrb(rscratch1, Address(rscratch1, ConstMethod::result_type_offset()));
58
59 Label done, notBool, notByte, notChar;
193 ldrw(index, Address(rbcp, bcp_offset));
194 } else if (index_size == sizeof(u1)) {
195 load_unsigned_byte(index, Address(rbcp, bcp_offset));
196 } else {
197 ShouldNotReachHere();
198 }
199 }
200
201 void InterpreterMacroAssembler::get_method_counters(Register method,
202 Register mcs, Label& skip) {
203 Label has_counters;
204 ldr(mcs, Address(method, Method::method_counters_offset()));
205 cbnz(mcs, has_counters);
206 call_VM(noreg, CAST_FROM_FN_PTR(address,
207 InterpreterRuntime::build_method_counters), method);
208 ldr(mcs, Address(method, Method::method_counters_offset()));
209 cbz(mcs, skip); // No MethodCounters allocated, OutOfMemory
210 bind(has_counters);
211 }
212
213 void InterpreterMacroAssembler::allocate_instance(Register klass, Register new_obj,
214 Register t1, Register t2,
215 bool clear_fields, Label& alloc_failed) {
216 MacroAssembler::allocate_instance(klass, new_obj, t1, t2, clear_fields, alloc_failed);
217 if (DTraceAllocProbes) {
218 // Trigger dtrace event for fastpath
219 push(atos);
220 call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), new_obj);
221 pop(atos);
222 }
223 }
224
225 void InterpreterMacroAssembler::read_flat_field(Register entry, Register obj) {
226 call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flat_field), obj, entry);
227 membar(Assembler::StoreStore);
228 }
229
230 void InterpreterMacroAssembler::write_flat_field(Register entry, Register field_offset,
231 Register tmp1, Register tmp2,
232 Register obj) {
233 assert_different_registers(entry, field_offset, tmp1, tmp2, obj);
234 Label slow_path, done;
235
236 load_unsigned_byte(tmp1, Address(entry, in_bytes(ResolvedFieldEntry::flags_offset())));
237 test_field_is_not_null_free_inline_type(tmp1, noreg /* temp */, slow_path);
238
239 null_check(r0); // FIXME JDK-8341120
240
241 add(obj, obj, field_offset);
242
243 load_klass(tmp1, r0);
244 payload_address(r0, r0, tmp1);
245
246 Register layout_info = field_offset;
247 load_unsigned_short(tmp1, Address(entry, in_bytes(ResolvedFieldEntry::field_index_offset())));
248 ldr(tmp2, Address(entry, in_bytes(ResolvedFieldEntry::field_holder_offset())));
249 inline_layout_info(tmp2, tmp1, layout_info);
250
251 flat_field_copy(IN_HEAP, r0, obj, layout_info);
252 b(done);
253
254 bind(slow_path);
255 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_flat_field), obj, r0, entry);
256 bind(done);
257 }
258
259 // Load object from cpool->resolved_references(index)
260 void InterpreterMacroAssembler::load_resolved_reference_at_index(
261 Register result, Register index, Register tmp) {
262 assert_different_registers(result, index);
263
264 get_constant_pool(result);
265 // load pointer for resolved_references[] objArray
266 ldr(result, Address(result, ConstantPool::cache_offset()));
267 ldr(result, Address(result, ConstantPoolCache::resolved_references_offset()));
268 resolve_oop_handle(result, tmp, rscratch2);
269 // Add in the index
270 add(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
271 load_heap_oop(result, Address(result, index, Address::uxtw(LogBytesPerHeapOop)), tmp, rscratch2);
272 }
273
274 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
275 Register cpool, Register index, Register klass, Register temp) {
276 add(temp, cpool, index, LSL, LogBytesPerWord);
277 ldrh(temp, Address(temp, sizeof(ConstantPool))); // temp = resolved_klass_index
278 ldr(klass, Address(cpool, ConstantPool::resolved_klasses_offset())); // klass = cpool->_resolved_klasses
279 add(klass, klass, temp, LSL, LogBytesPerWord);
280 ldr(klass, Address(klass, Array<Klass*>::base_offset_in_bytes()));
281 }
282
283 // Generate a subtype check: branch to ok_is_subtype if sub_klass is a
284 // subtype of super_klass.
285 //
286 // Args:
287 // r0: superklass
288 // Rsub_klass: subklass
289 //
290 // Kills:
291 // r2
292 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
293 Label& ok_is_subtype,
294 bool profile) {
295 assert(Rsub_klass != r0, "r0 holds superklass");
296 assert(Rsub_klass != r2, "r2 holds 2ndary super array length");
297
298 // Profile the not-null value's klass.
299 if (profile) {
300 profile_typecheck(r2, Rsub_klass); // blows r2
301 }
302 // Do the check.
303 check_klass_subtype(Rsub_klass, r0, r2, ok_is_subtype); // blows r2
304 }
305
306 // Java Expression Stack
307
308 void InterpreterMacroAssembler::pop_ptr(Register r) {
309 ldr(r, post(esp, wordSize));
310 }
311
312 void InterpreterMacroAssembler::pop_i(Register r) {
313 ldrw(r, post(esp, wordSize));
314 }
315
316 void InterpreterMacroAssembler::pop_l(Register r) {
317 ldr(r, post(esp, 2 * Interpreter::stackElementSize));
318 }
319
320 void InterpreterMacroAssembler::push_ptr(Register r) {
321 str(r, pre(esp, -wordSize));
691 cmpw(rscratch1, (u1)StackOverflow::stack_guard_enabled);
692 br(Assembler::EQ, no_reserved_zone_enabling);
693
694 // look for an overflow into the stack reserved zone, i.e.
695 // interpreter_frame_sender_sp <= JavaThread::reserved_stack_activation
696 ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset()));
697 cmp(rscratch2, rscratch1);
698 br(Assembler::LS, no_reserved_zone_enabling);
699
700 JFR_ONLY(leave_jfr_critical_section();)
701
702 call_VM_leaf(
703 CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), rthread);
704 call_VM(noreg, CAST_FROM_FN_PTR(address,
705 InterpreterRuntime::throw_delayed_StackOverflowError));
706 should_not_reach_here();
707
708 bind(no_reserved_zone_enabling);
709 }
710
711 if (state == atos && InlineTypeReturnedAsFields) {
712 Label skip;
713 Label not_null;
714 cbnz(r0, not_null);
715 // Returned value is null, zero all return registers because they may belong to oop fields
716 mov(j_rarg1, zr);
717 mov(j_rarg2, zr);
718 mov(j_rarg3, zr);
719 mov(j_rarg4, zr);
720 mov(j_rarg5, zr);
721 mov(j_rarg6, zr);
722 mov(j_rarg7, zr);
723 b(skip);
724 bind(not_null);
725
726 // Check if we are returning an non-null inline type and load its fields into registers
727 test_oop_is_not_inline_type(r0, rscratch2, skip, /* can_be_null= */ false);
728
729 // Load fields from a buffered value with an inline class specific handler
730 load_klass(rscratch1 /*dst*/, r0 /*src*/);
731 ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
732 ldr(rscratch1, Address(rscratch1, InlineKlass::unpack_handler_offset()));
733 // Unpack handler can be null if inline type is not scalarizable in returns
734 cbz(rscratch1, skip);
735
736 blr(rscratch1);
737 #ifdef ASSERT
738 // TODO 8284443 Enable
739 if (StressCallingConvention && false) {
740 Label skip_stress;
741 ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
742 ldrw(rscratch1, Address(rscratch1, Method::flags_offset()));
743 tstw(rscratch1, MethodFlags::has_scalarized_return_flag());
744 br(Assembler::EQ, skip_stress);
745 load_klass(r0, r0);
746 orr(r0, r0, 1);
747 bind(skip_stress);
748 }
749 #endif
750 bind(skip);
751 // Check above kills sender esp in rscratch2. Reload it.
752 ldr(rscratch2, Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
753 }
754
755 // remove frame anchor
756 leave();
757
758 JFR_ONLY(leave_jfr_critical_section();)
759
760 // restore sender esp
761 mov(esp, rscratch2);
762
763 // If we're returning to interpreted code we will shortly be
764 // adjusting SP to allow some space for ESP. If we're returning to
765 // compiled code the saved sender SP was saved in sender_sp, so this
766 // restores it.
767 andr(sp, esp, -16);
768 }
769
770 #if INCLUDE_JFR
771 void InterpreterMacroAssembler::enter_jfr_critical_section() {
772 const Address sampling_critical_section(rthread, in_bytes(SAMPLING_CRITICAL_SECTION_OFFSET_JFR));
773 mov(rscratch1, true);
774 strb(rscratch1, sampling_critical_section);
1012 }
1013
1014
1015 void InterpreterMacroAssembler::profile_taken_branch(Register mdp) {
1016 if (ProfileInterpreter) {
1017 Label profile_continue;
1018
1019 // If no method data exists, go to profile_continue.
1020 test_method_data_pointer(mdp, profile_continue);
1021
1022 // We are taking a branch. Increment the taken count.
1023 increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
1024
1025 // The method data pointer needs to be updated to reflect the new target.
1026 update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
1027 bind(profile_continue);
1028 }
1029 }
1030
1031
1032 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp, bool acmp) {
1033 if (ProfileInterpreter) {
1034 Label profile_continue;
1035
1036 // If no method data exists, go to profile_continue.
1037 test_method_data_pointer(mdp, profile_continue);
1038
1039 // We are not taking a branch. Increment the not taken count.
1040 increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
1041
1042 // The method data pointer needs to be updated to correspond to
1043 // the next bytecode
1044 update_mdp_by_constant(mdp, acmp ? in_bytes(ACmpData::acmp_data_size()) : in_bytes(BranchData::branch_data_size()));
1045 bind(profile_continue);
1046 }
1047 }
1048
1049
1050 void InterpreterMacroAssembler::profile_call(Register mdp) {
1051 if (ProfileInterpreter) {
1052 Label profile_continue;
1053
1054 // If no method data exists, go to profile_continue.
1055 test_method_data_pointer(mdp, profile_continue);
1056
1057 // We are making a call. Increment the count.
1058 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1059
1060 // The method data pointer needs to be updated to reflect the new target.
1061 update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1062 bind(profile_continue);
1063 }
1064 }
1210 // case_array_offset_in_bytes()
1211 movw(reg2, in_bytes(MultiBranchData::per_case_size()));
1212 movw(rscratch1, in_bytes(MultiBranchData::case_array_offset()));
1213 Assembler::maddw(index, index, reg2, rscratch1);
1214
1215 // Update the case count
1216 increment_mdp_data_at(mdp,
1217 index,
1218 in_bytes(MultiBranchData::relative_count_offset()));
1219
1220 // The method data pointer needs to be updated.
1221 update_mdp_by_offset(mdp,
1222 index,
1223 in_bytes(MultiBranchData::
1224 relative_displacement_offset()));
1225
1226 bind(profile_continue);
1227 }
1228 }
1229
1230 template <class ArrayData> void InterpreterMacroAssembler::profile_array_type(Register mdp,
1231 Register array,
1232 Register tmp) {
1233 if (ProfileInterpreter) {
1234 Label profile_continue;
1235
1236 // If no method data exists, go to profile_continue.
1237 test_method_data_pointer(mdp, profile_continue);
1238
1239 mov(tmp, array);
1240 profile_obj_type(tmp, Address(mdp, in_bytes(ArrayData::array_offset())));
1241
1242 Label not_flat;
1243 test_non_flat_array_oop(array, tmp, not_flat);
1244
1245 set_mdp_flag_at(mdp, ArrayData::flat_array_byte_constant());
1246
1247 bind(not_flat);
1248
1249 Label not_null_free;
1250 test_non_null_free_array_oop(array, tmp, not_null_free);
1251
1252 set_mdp_flag_at(mdp, ArrayData::null_free_array_byte_constant());
1253
1254 bind(not_null_free);
1255
1256 bind(profile_continue);
1257 }
1258 }
1259
1260 template void InterpreterMacroAssembler::profile_array_type<ArrayLoadData>(Register mdp,
1261 Register array,
1262 Register tmp);
1263 template void InterpreterMacroAssembler::profile_array_type<ArrayStoreData>(Register mdp,
1264 Register array,
1265 Register tmp);
1266
1267 void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2) {
1268 if (ProfileInterpreter) {
1269 Label profile_continue;
1270
1271 // If no method data exists, go to profile_continue.
1272 test_method_data_pointer(mdp, profile_continue);
1273
1274 Label done, update;
1275 cbnz(element, update);
1276 set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
1277 b(done);
1278
1279 bind(update);
1280 load_klass(tmp, element);
1281
1282 // Record the object type.
1283 profile_receiver_type(tmp, mdp, 0);
1284
1285 bind(done);
1286
1287 // The method data pointer needs to be updated.
1288 update_mdp_by_constant(mdp, in_bytes(ArrayStoreData::array_store_data_size()));
1289
1290 bind(profile_continue);
1291 }
1292 }
1293
1294
1295 void InterpreterMacroAssembler::profile_element_type(Register mdp,
1296 Register element,
1297 Register tmp) {
1298 if (ProfileInterpreter) {
1299 Label profile_continue;
1300
1301 // If no method data exists, go to profile_continue.
1302 test_method_data_pointer(mdp, profile_continue);
1303
1304 mov(tmp, element);
1305 profile_obj_type(tmp, Address(mdp, in_bytes(ArrayLoadData::element_offset())));
1306
1307 // The method data pointer needs to be updated.
1308 update_mdp_by_constant(mdp, in_bytes(ArrayLoadData::array_load_data_size()));
1309
1310 bind(profile_continue);
1311 }
1312 }
1313
1314 void InterpreterMacroAssembler::profile_acmp(Register mdp,
1315 Register left,
1316 Register right,
1317 Register tmp) {
1318 if (ProfileInterpreter) {
1319 Label profile_continue;
1320
1321 // If no method data exists, go to profile_continue.
1322 test_method_data_pointer(mdp, profile_continue);
1323
1324 mov(tmp, left);
1325 profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::left_offset())));
1326
1327 Label left_not_inline_type;
1328 test_oop_is_not_inline_type(left, tmp, left_not_inline_type);
1329 set_mdp_flag_at(mdp, ACmpData::left_inline_type_byte_constant());
1330 bind(left_not_inline_type);
1331
1332 mov(tmp, right);
1333 profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::right_offset())));
1334
1335 Label right_not_inline_type;
1336 test_oop_is_not_inline_type(right, tmp, right_not_inline_type);
1337 set_mdp_flag_at(mdp, ACmpData::right_inline_type_byte_constant());
1338 bind(right_not_inline_type);
1339
1340 bind(profile_continue);
1341 }
1342 }
1343
1344 void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) {
1345 if (state == atos) {
1346 MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line);
1347 }
1348 }
1349
1350 void InterpreterMacroAssembler::notify_method_entry() {
1351 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1352 // track stack depth. If it is possible to enter interp_only_mode we add
1353 // the code to check if the event should be sent.
1354 if (JvmtiExport::can_post_interpreter_events()) {
1355 Label L;
1356 ldrw(r3, Address(rthread, JavaThread::interp_only_mode_offset()));
1357 cbzw(r3, L);
1358 call_VM(noreg, CAST_FROM_FN_PTR(address,
1359 InterpreterRuntime::post_method_entry));
1360 bind(L);
1361 }
1362
1363 if (DTraceMethodProbes) {
1687 profile_obj_type(tmp, mdo_arg_addr);
1688
1689 int to_add = in_bytes(TypeStackSlotEntries::per_arg_size());
1690 off_to_args += to_add;
1691 }
1692
1693 if (MethodData::profile_return()) {
1694 ldr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())));
1695 sub(tmp, tmp, TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count());
1696 }
1697
1698 add(rscratch1, mdp, off_to_args);
1699 bind(done);
1700 mov(mdp, rscratch1);
1701
1702 if (MethodData::profile_return()) {
1703 // We're right after the type profile for the last
1704 // argument. tmp is the number of cells left in the
1705 // CallTypeData/VirtualCallTypeData to reach its end. Non null
1706 // if there's a return to profile.
1707 assert(SingleTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
1708 add(mdp, mdp, tmp, LSL, exact_log2(DataLayout::cell_size));
1709 }
1710 str(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1711 } else {
1712 assert(MethodData::profile_return(), "either profile call args or call ret");
1713 update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size()));
1714 }
1715
1716 // mdp points right after the end of the
1717 // CallTypeData/VirtualCallTypeData, right after the cells for the
1718 // return value type if there's one
1719
1720 bind(profile_continue);
1721 }
1722 }
1723
1724 void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) {
1725 assert_different_registers(mdp, ret, tmp, rbcp);
1726 if (ProfileInterpreter && MethodData::profile_return()) {
1727 Label profile_continue, done;
1733
1734 // If we don't profile all invoke bytecodes we must make sure
1735 // it's a bytecode we indeed profile. We can't go back to the
1736 // beginning of the ProfileData we intend to update to check its
1737 // type because we're right after it and we don't known its
1738 // length
1739 Label do_profile;
1740 ldrb(rscratch1, Address(rbcp, 0));
1741 cmp(rscratch1, (u1)Bytecodes::_invokedynamic);
1742 br(Assembler::EQ, do_profile);
1743 cmp(rscratch1, (u1)Bytecodes::_invokehandle);
1744 br(Assembler::EQ, do_profile);
1745 get_method(tmp);
1746 ldrh(rscratch1, Address(tmp, Method::intrinsic_id_offset()));
1747 subs(zr, rscratch1, static_cast<int>(vmIntrinsics::_compiledLambdaForm));
1748 br(Assembler::NE, profile_continue);
1749
1750 bind(do_profile);
1751 }
1752
1753 Address mdo_ret_addr(mdp, -in_bytes(SingleTypeEntry::size()));
1754 mov(tmp, ret);
1755 profile_obj_type(tmp, mdo_ret_addr);
1756
1757 bind(profile_continue);
1758 }
1759 }
1760
1761 void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2) {
1762 assert_different_registers(rscratch1, rscratch2, mdp, tmp1, tmp2);
1763 if (ProfileInterpreter && MethodData::profile_parameters()) {
1764 Label profile_continue, done;
1765
1766 test_method_data_pointer(mdp, profile_continue);
1767
1768 // Load the offset of the area within the MDO used for
1769 // parameters. If it's negative we're not profiling any parameters
1770 ldrw(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset())));
1771 tbnz(tmp1, 31, profile_continue); // i.e. sign bit set
1772
1773 // Compute a pointer to the area for parameters from the offset
|