15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "compiler/compiler_globals.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/barrierSetAssembler.hpp"
30 #include "interp_masm_aarch64.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interpreterRuntime.hpp"
33 #include "logging/log.hpp"
34 #include "oops/arrayOop.hpp"
35 #include "oops/markWord.hpp"
36 #include "oops/method.hpp"
37 #include "oops/methodData.hpp"
38 #include "oops/resolvedFieldEntry.hpp"
39 #include "oops/resolvedIndyEntry.hpp"
40 #include "oops/resolvedMethodEntry.hpp"
41 #include "prims/jvmtiExport.hpp"
42 #include "prims/jvmtiThreadState.hpp"
43 #include "runtime/basicLock.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/javaThread.hpp"
46 #include "runtime/safepointMechanism.hpp"
47 #include "runtime/sharedRuntime.hpp"
48 #include "utilities/powerOfTwo.hpp"
49
50 void InterpreterMacroAssembler::narrow(Register result) {
51
52 // Get method->_constMethod->_result_type
53 ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
54 ldr(rscratch1, Address(rscratch1, Method::const_offset()));
55 ldrb(rscratch1, Address(rscratch1, ConstMethod::result_type_offset()));
56
57 Label done, notBool, notByte, notChar;
191 ldrw(index, Address(rbcp, bcp_offset));
192 } else if (index_size == sizeof(u1)) {
193 load_unsigned_byte(index, Address(rbcp, bcp_offset));
194 } else {
195 ShouldNotReachHere();
196 }
197 }
198
199 void InterpreterMacroAssembler::get_method_counters(Register method,
200 Register mcs, Label& skip) {
201 Label has_counters;
202 ldr(mcs, Address(method, Method::method_counters_offset()));
203 cbnz(mcs, has_counters);
204 call_VM(noreg, CAST_FROM_FN_PTR(address,
205 InterpreterRuntime::build_method_counters), method);
206 ldr(mcs, Address(method, Method::method_counters_offset()));
207 cbz(mcs, skip); // No MethodCounters allocated, OutOfMemory
208 bind(has_counters);
209 }
210
211 // Load object from cpool->resolved_references(index)
212 void InterpreterMacroAssembler::load_resolved_reference_at_index(
213 Register result, Register index, Register tmp) {
214 assert_different_registers(result, index);
215
216 get_constant_pool(result);
217 // load pointer for resolved_references[] objArray
218 ldr(result, Address(result, ConstantPool::cache_offset()));
219 ldr(result, Address(result, ConstantPoolCache::resolved_references_offset()));
220 resolve_oop_handle(result, tmp, rscratch2);
221 // Add in the index
222 add(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
223 load_heap_oop(result, Address(result, index, Address::uxtw(LogBytesPerHeapOop)), tmp, rscratch2);
224 }
225
226 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
227 Register cpool, Register index, Register klass, Register temp) {
228 add(temp, cpool, index, LSL, LogBytesPerWord);
229 ldrh(temp, Address(temp, sizeof(ConstantPool))); // temp = resolved_klass_index
230 ldr(klass, Address(cpool, ConstantPool::resolved_klasses_offset())); // klass = cpool->_resolved_klasses
231 add(klass, klass, temp, LSL, LogBytesPerWord);
232 ldr(klass, Address(klass, Array<Klass*>::base_offset_in_bytes()));
233 }
234
235 // Generate a subtype check: branch to ok_is_subtype if sub_klass is a
236 // subtype of super_klass.
237 //
238 // Args:
239 // r0: superklass
240 // Rsub_klass: subklass
241 //
242 // Kills:
243 // r2, r5
244 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
245 Label& ok_is_subtype) {
246 assert(Rsub_klass != r0, "r0 holds superklass");
247 assert(Rsub_klass != r2, "r2 holds 2ndary super array length");
248 assert(Rsub_klass != r5, "r5 holds 2ndary super array scan ptr");
249
250 // Profile the not-null value's klass.
251 profile_typecheck(r2, Rsub_klass, r5); // blows r2, reloads r5
252
253 // Do the check.
254 check_klass_subtype(Rsub_klass, r0, r2, ok_is_subtype); // blows r2
255 }
256
257 // Java Expression Stack
258
259 void InterpreterMacroAssembler::pop_ptr(Register r) {
260 ldr(r, post(esp, wordSize));
261 }
262
263 void InterpreterMacroAssembler::pop_i(Register r) {
264 ldrw(r, post(esp, wordSize));
265 }
266
267 void InterpreterMacroAssembler::pop_l(Register r) {
268 ldr(r, post(esp, 2 * Interpreter::stackElementSize));
269 }
270
271 void InterpreterMacroAssembler::push_ptr(Register r) {
615 br(Assembler::AL, fast_path);
616 bind(slow_path);
617 push(state);
618 set_last_Java_frame(esp, rfp, pc(), rscratch1);
619 super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), rthread);
620 reset_last_Java_frame(true);
621 pop(state);
622 bind(fast_path);
623
624 // JVMTI support. Make sure the safepoint poll test is issued prior.
625 if (notify_jvmdi) {
626 notify_method_exit(state, NotifyJVMTI); // preserve TOSCA
627 } else {
628 notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
629 }
630
631 // remove activation
632 // get sender esp
633 ldr(rscratch2,
634 Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
635 if (StackReservedPages > 0) {
636 // testing if reserved zone needs to be re-enabled
637 Label no_reserved_zone_enabling;
638
639 // check if already enabled - if so no re-enabling needed
640 assert(sizeof(StackOverflow::StackGuardState) == 4, "unexpected size");
641 ldrw(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
642 cmpw(rscratch1, (u1)StackOverflow::stack_guard_enabled);
643 br(Assembler::EQ, no_reserved_zone_enabling);
644
645 // look for an overflow into the stack reserved zone, i.e.
646 // interpreter_frame_sender_sp <= JavaThread::reserved_stack_activation
647 ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset()));
648 cmp(rscratch2, rscratch1);
649 br(Assembler::LS, no_reserved_zone_enabling);
650
651 JFR_ONLY(leave_jfr_critical_section();)
652
653 call_VM_leaf(
654 CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), rthread);
655 call_VM(noreg, CAST_FROM_FN_PTR(address,
656 InterpreterRuntime::throw_delayed_StackOverflowError));
657 should_not_reach_here();
658
659 bind(no_reserved_zone_enabling);
660 }
661
662 // remove frame anchor
663 leave();
664
665 JFR_ONLY(leave_jfr_critical_section();)
666
667 // restore sender esp
668 mov(esp, rscratch2);
669
670 // If we're returning to interpreted code we will shortly be
671 // adjusting SP to allow some space for ESP. If we're returning to
672 // compiled code the saved sender SP was saved in sender_sp, so this
673 // restores it.
674 andr(sp, esp, -16);
675 }
676
677 #if INCLUDE_JFR
678 void InterpreterMacroAssembler::enter_jfr_critical_section() {
679 const Address sampling_critical_section(rthread, in_bytes(SAMPLING_CRITICAL_SECTION_OFFSET_JFR));
680 mov(rscratch1, true);
681 strb(rscratch1, sampling_critical_section);
919 }
920
921
922 void InterpreterMacroAssembler::profile_taken_branch(Register mdp) {
923 if (ProfileInterpreter) {
924 Label profile_continue;
925
926 // If no method data exists, go to profile_continue.
927 test_method_data_pointer(mdp, profile_continue);
928
929 // We are taking a branch. Increment the taken count.
930 increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
931
932 // The method data pointer needs to be updated to reflect the new target.
933 update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
934 bind(profile_continue);
935 }
936 }
937
938
939 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
940 if (ProfileInterpreter) {
941 Label profile_continue;
942
943 // If no method data exists, go to profile_continue.
944 test_method_data_pointer(mdp, profile_continue);
945
946 // We are not taking a branch. Increment the not taken count.
947 increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
948
949 // The method data pointer needs to be updated to correspond to
950 // the next bytecode
951 update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));
952 bind(profile_continue);
953 }
954 }
955
956
957 void InterpreterMacroAssembler::profile_call(Register mdp) {
958 if (ProfileInterpreter) {
959 Label profile_continue;
960
961 // If no method data exists, go to profile_continue.
962 test_method_data_pointer(mdp, profile_continue);
963
964 // We are making a call. Increment the count.
965 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
966
967 // The method data pointer needs to be updated to reflect the new target.
968 update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
969 bind(profile_continue);
970 }
971 }
1254 // case_array_offset_in_bytes()
1255 movw(reg2, in_bytes(MultiBranchData::per_case_size()));
1256 movw(rscratch1, in_bytes(MultiBranchData::case_array_offset()));
1257 Assembler::maddw(index, index, reg2, rscratch1);
1258
1259 // Update the case count
1260 increment_mdp_data_at(mdp,
1261 index,
1262 in_bytes(MultiBranchData::relative_count_offset()));
1263
1264 // The method data pointer needs to be updated.
1265 update_mdp_by_offset(mdp,
1266 index,
1267 in_bytes(MultiBranchData::
1268 relative_displacement_offset()));
1269
1270 bind(profile_continue);
1271 }
1272 }
1273
1274 void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) {
1275 if (state == atos) {
1276 MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line);
1277 }
1278 }
1279
1280 void InterpreterMacroAssembler::notify_method_entry() {
1281 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1282 // track stack depth. If it is possible to enter interp_only_mode we add
1283 // the code to check if the event should be sent.
1284 if (JvmtiExport::can_post_interpreter_events()) {
1285 Label L;
1286 ldrw(r3, Address(rthread, JavaThread::interp_only_mode_offset()));
1287 cbzw(r3, L);
1288 call_VM(noreg, CAST_FROM_FN_PTR(address,
1289 InterpreterRuntime::post_method_entry));
1290 bind(L);
1291 }
1292
1293 if (DTraceMethodProbes) {
1617 profile_obj_type(tmp, mdo_arg_addr);
1618
1619 int to_add = in_bytes(TypeStackSlotEntries::per_arg_size());
1620 off_to_args += to_add;
1621 }
1622
1623 if (MethodData::profile_return()) {
1624 ldr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())));
1625 sub(tmp, tmp, TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count());
1626 }
1627
1628 add(rscratch1, mdp, off_to_args);
1629 bind(done);
1630 mov(mdp, rscratch1);
1631
1632 if (MethodData::profile_return()) {
1633 // We're right after the type profile for the last
1634 // argument. tmp is the number of cells left in the
1635 // CallTypeData/VirtualCallTypeData to reach its end. Non null
1636 // if there's a return to profile.
1637 assert(ReturnTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
1638 add(mdp, mdp, tmp, LSL, exact_log2(DataLayout::cell_size));
1639 }
1640 str(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1641 } else {
1642 assert(MethodData::profile_return(), "either profile call args or call ret");
1643 update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size()));
1644 }
1645
1646 // mdp points right after the end of the
1647 // CallTypeData/VirtualCallTypeData, right after the cells for the
1648 // return value type if there's one
1649
1650 bind(profile_continue);
1651 }
1652 }
1653
1654 void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) {
1655 assert_different_registers(mdp, ret, tmp, rbcp);
1656 if (ProfileInterpreter && MethodData::profile_return()) {
1657 Label profile_continue, done;
1663
1664 // If we don't profile all invoke bytecodes we must make sure
1665 // it's a bytecode we indeed profile. We can't go back to the
1666 // beginning of the ProfileData we intend to update to check its
1667 // type because we're right after it and we don't known its
1668 // length
1669 Label do_profile;
1670 ldrb(rscratch1, Address(rbcp, 0));
1671 cmp(rscratch1, (u1)Bytecodes::_invokedynamic);
1672 br(Assembler::EQ, do_profile);
1673 cmp(rscratch1, (u1)Bytecodes::_invokehandle);
1674 br(Assembler::EQ, do_profile);
1675 get_method(tmp);
1676 ldrh(rscratch1, Address(tmp, Method::intrinsic_id_offset()));
1677 subs(zr, rscratch1, static_cast<int>(vmIntrinsics::_compiledLambdaForm));
1678 br(Assembler::NE, profile_continue);
1679
1680 bind(do_profile);
1681 }
1682
1683 Address mdo_ret_addr(mdp, -in_bytes(ReturnTypeEntry::size()));
1684 mov(tmp, ret);
1685 profile_obj_type(tmp, mdo_ret_addr);
1686
1687 bind(profile_continue);
1688 }
1689 }
1690
1691 void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2) {
1692 assert_different_registers(rscratch1, rscratch2, mdp, tmp1, tmp2);
1693 if (ProfileInterpreter && MethodData::profile_parameters()) {
1694 Label profile_continue, done;
1695
1696 test_method_data_pointer(mdp, profile_continue);
1697
1698 // Load the offset of the area within the MDO used for
1699 // parameters. If it's negative we're not profiling any parameters
1700 ldrw(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset())));
1701 tbnz(tmp1, 31, profile_continue); // i.e. sign bit set
1702
1703 // Compute a pointer to the area for parameters from the offset
|
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "compiler/compiler_globals.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/barrierSetAssembler.hpp"
30 #include "interp_masm_aarch64.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interpreterRuntime.hpp"
33 #include "logging/log.hpp"
34 #include "oops/arrayOop.hpp"
35 #include "oops/constMethodFlags.hpp"
36 #include "oops/markWord.hpp"
37 #include "oops/method.hpp"
38 #include "oops/methodData.hpp"
39 #include "oops/inlineKlass.hpp"
40 #include "oops/resolvedFieldEntry.hpp"
41 #include "oops/resolvedIndyEntry.hpp"
42 #include "oops/resolvedMethodEntry.hpp"
43 #include "prims/jvmtiExport.hpp"
44 #include "prims/jvmtiThreadState.hpp"
45 #include "runtime/basicLock.hpp"
46 #include "runtime/frame.inline.hpp"
47 #include "runtime/javaThread.hpp"
48 #include "runtime/safepointMechanism.hpp"
49 #include "runtime/sharedRuntime.hpp"
50 #include "utilities/powerOfTwo.hpp"
51
52 void InterpreterMacroAssembler::narrow(Register result) {
53
54 // Get method->_constMethod->_result_type
55 ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
56 ldr(rscratch1, Address(rscratch1, Method::const_offset()));
57 ldrb(rscratch1, Address(rscratch1, ConstMethod::result_type_offset()));
58
59 Label done, notBool, notByte, notChar;
193 ldrw(index, Address(rbcp, bcp_offset));
194 } else if (index_size == sizeof(u1)) {
195 load_unsigned_byte(index, Address(rbcp, bcp_offset));
196 } else {
197 ShouldNotReachHere();
198 }
199 }
200
201 void InterpreterMacroAssembler::get_method_counters(Register method,
202 Register mcs, Label& skip) {
203 Label has_counters;
204 ldr(mcs, Address(method, Method::method_counters_offset()));
205 cbnz(mcs, has_counters);
206 call_VM(noreg, CAST_FROM_FN_PTR(address,
207 InterpreterRuntime::build_method_counters), method);
208 ldr(mcs, Address(method, Method::method_counters_offset()));
209 cbz(mcs, skip); // No MethodCounters allocated, OutOfMemory
210 bind(has_counters);
211 }
212
213 void InterpreterMacroAssembler::allocate_instance(Register klass, Register new_obj,
214 Register t1, Register t2,
215 bool clear_fields, Label& alloc_failed) {
216 MacroAssembler::allocate_instance(klass, new_obj, t1, t2, clear_fields, alloc_failed);
217 if (DTraceAllocProbes) {
218 // Trigger dtrace event for fastpath
219 push(atos);
220 call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), new_obj);
221 pop(atos);
222 }
223 }
224
225 void InterpreterMacroAssembler::read_flat_field(Register entry,
226 Register field_index, Register field_offset,
227 Register temp, Register obj) {
228 Label failed_alloc, slow_path, done;
229 const Register src = field_offset;
230 const Register alloc_temp = r10;
231 const Register dst_temp = field_index;
232 const Register layout_info = temp;
233 assert_different_registers(obj, entry, field_index, field_offset, temp, alloc_temp, rscratch1);
234
235 load_unsigned_byte(temp, Address(entry, in_bytes(ResolvedFieldEntry::flags_offset())));
236 // If the field is nullable, jump to slow path
237 tbz(temp, ResolvedFieldEntry::is_null_free_inline_type_shift, slow_path);
238
239 // Grab the inline field klass
240 ldr(rscratch1, Address(entry, in_bytes(ResolvedFieldEntry::field_holder_offset())));
241 inline_layout_info(rscratch1, field_index, layout_info);
242
243 const Register field_klass = dst_temp;
244 ldr(field_klass, Address(layout_info, in_bytes(InlineLayoutInfo::klass_offset())));
245
246 // allocate buffer
247 push(obj); // save holder
248 allocate_instance(field_klass, obj, alloc_temp, rscratch2, false, failed_alloc);
249
250 // Have an oop instance buffer, copy into it
251 payload_address(obj, dst_temp, field_klass); // danger, uses rscratch1
252 pop(alloc_temp); // restore holder
253 lea(src, Address(alloc_temp, field_offset));
254 // call_VM_leaf, clobbers a few regs, save restore new obj
255 push(obj);
256 flat_field_copy(IS_DEST_UNINITIALIZED, src, dst_temp, layout_info);
257 pop(obj);
258 b(done);
259
260 bind(failed_alloc);
261 pop(obj);
262 bind(slow_path);
263 call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flat_field),
264 obj, entry);
265
266 bind(done);
267 membar(Assembler::StoreStore);
268 }
269
270 void InterpreterMacroAssembler::write_flat_field(Register entry, Register field_offset,
271 Register tmp1, Register tmp2,
272 Register obj) {
273 assert_different_registers(entry, field_offset, tmp1, tmp2, obj);
274 Label slow_path, done;
275
276 load_unsigned_byte(tmp1, Address(entry, in_bytes(ResolvedFieldEntry::flags_offset())));
277 test_field_is_not_null_free_inline_type(tmp1, noreg /* temp */, slow_path);
278
279 null_check(r0); // FIXME JDK-8341120
280
281 add(obj, obj, field_offset);
282
283 load_klass(tmp1, r0);
284 payload_address(r0, r0, tmp1);
285
286 Register layout_info = field_offset;
287 load_unsigned_short(tmp1, Address(entry, in_bytes(ResolvedFieldEntry::field_index_offset())));
288 ldr(tmp2, Address(entry, in_bytes(ResolvedFieldEntry::field_holder_offset())));
289 inline_layout_info(tmp2, tmp1, layout_info);
290
291 flat_field_copy(IN_HEAP, r0, obj, layout_info);
292 b(done);
293
294 bind(slow_path);
295 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_flat_field), obj, r0, entry);
296 bind(done);
297 }
298
299 // Load object from cpool->resolved_references(index)
300 void InterpreterMacroAssembler::load_resolved_reference_at_index(
301 Register result, Register index, Register tmp) {
302 assert_different_registers(result, index);
303
304 get_constant_pool(result);
305 // load pointer for resolved_references[] objArray
306 ldr(result, Address(result, ConstantPool::cache_offset()));
307 ldr(result, Address(result, ConstantPoolCache::resolved_references_offset()));
308 resolve_oop_handle(result, tmp, rscratch2);
309 // Add in the index
310 add(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
311 load_heap_oop(result, Address(result, index, Address::uxtw(LogBytesPerHeapOop)), tmp, rscratch2);
312 }
313
314 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
315 Register cpool, Register index, Register klass, Register temp) {
316 add(temp, cpool, index, LSL, LogBytesPerWord);
317 ldrh(temp, Address(temp, sizeof(ConstantPool))); // temp = resolved_klass_index
318 ldr(klass, Address(cpool, ConstantPool::resolved_klasses_offset())); // klass = cpool->_resolved_klasses
319 add(klass, klass, temp, LSL, LogBytesPerWord);
320 ldr(klass, Address(klass, Array<Klass*>::base_offset_in_bytes()));
321 }
322
323 // Generate a subtype check: branch to ok_is_subtype if sub_klass is a
324 // subtype of super_klass.
325 //
326 // Args:
327 // r0: superklass
328 // Rsub_klass: subklass
329 //
330 // Kills:
331 // r2, r5
332 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
333 Label& ok_is_subtype,
334 bool profile) {
335 assert(Rsub_klass != r0, "r0 holds superklass");
336 assert(Rsub_klass != r2, "r2 holds 2ndary super array length");
337 assert(Rsub_klass != r5, "r5 holds 2ndary super array scan ptr");
338
339 // Profile the not-null value's klass.
340 if (profile) {
341 profile_typecheck(r2, Rsub_klass, r5); // blows r2, reloads r5
342 }
343
344 // Do the check.
345 check_klass_subtype(Rsub_klass, r0, r2, ok_is_subtype); // blows r2
346 }
347
348 // Java Expression Stack
349
350 void InterpreterMacroAssembler::pop_ptr(Register r) {
351 ldr(r, post(esp, wordSize));
352 }
353
354 void InterpreterMacroAssembler::pop_i(Register r) {
355 ldrw(r, post(esp, wordSize));
356 }
357
358 void InterpreterMacroAssembler::pop_l(Register r) {
359 ldr(r, post(esp, 2 * Interpreter::stackElementSize));
360 }
361
362 void InterpreterMacroAssembler::push_ptr(Register r) {
706 br(Assembler::AL, fast_path);
707 bind(slow_path);
708 push(state);
709 set_last_Java_frame(esp, rfp, pc(), rscratch1);
710 super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), rthread);
711 reset_last_Java_frame(true);
712 pop(state);
713 bind(fast_path);
714
715 // JVMTI support. Make sure the safepoint poll test is issued prior.
716 if (notify_jvmdi) {
717 notify_method_exit(state, NotifyJVMTI); // preserve TOSCA
718 } else {
719 notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
720 }
721
722 // remove activation
723 // get sender esp
724 ldr(rscratch2,
725 Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
726
727 if (StackReservedPages > 0) {
728 // testing if reserved zone needs to be re-enabled
729 Label no_reserved_zone_enabling;
730
731 // check if already enabled - if so no re-enabling needed
732 assert(sizeof(StackOverflow::StackGuardState) == 4, "unexpected size");
733 ldrw(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
734 cmpw(rscratch1, (u1)StackOverflow::stack_guard_enabled);
735 br(Assembler::EQ, no_reserved_zone_enabling);
736
737 // look for an overflow into the stack reserved zone, i.e.
738 // interpreter_frame_sender_sp <= JavaThread::reserved_stack_activation
739 ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset()));
740 cmp(rscratch2, rscratch1);
741 br(Assembler::LS, no_reserved_zone_enabling);
742
743 JFR_ONLY(leave_jfr_critical_section();)
744
745 call_VM_leaf(
746 CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), rthread);
747 call_VM(noreg, CAST_FROM_FN_PTR(address,
748 InterpreterRuntime::throw_delayed_StackOverflowError));
749 should_not_reach_here();
750
751 bind(no_reserved_zone_enabling);
752 }
753
754 if (state == atos && InlineTypeReturnedAsFields) {
755 Label skip;
756 Label not_null;
757 cbnz(r0, not_null);
758 // Returned value is null, zero all return registers because they may belong to oop fields
759 mov(j_rarg1, zr);
760 mov(j_rarg2, zr);
761 mov(j_rarg3, zr);
762 mov(j_rarg4, zr);
763 mov(j_rarg5, zr);
764 mov(j_rarg6, zr);
765 mov(j_rarg7, zr);
766 b(skip);
767 bind(not_null);
768
769 // Check if we are returning an non-null inline type and load its fields into registers
770 test_oop_is_not_inline_type(r0, rscratch2, skip, /* can_be_null= */ false);
771
772 // Load fields from a buffered value with an inline class specific handler
773 load_klass(rscratch1 /*dst*/, r0 /*src*/);
774 ldr(rscratch1, Address(rscratch1, InstanceKlass::adr_inlineklass_fixed_block_offset()));
775 ldr(rscratch1, Address(rscratch1, InlineKlass::unpack_handler_offset()));
776 // Unpack handler can be null if inline type is not scalarizable in returns
777 cbz(rscratch1, skip);
778
779 blr(rscratch1);
780 #ifdef ASSERT
781 // TODO 8284443 Enable
782 if (StressCallingConvention && false) {
783 Label skip_stress;
784 ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
785 ldrw(rscratch1, Address(rscratch1, Method::flags_offset()));
786 tstw(rscratch1, MethodFlags::has_scalarized_return_flag());
787 br(Assembler::EQ, skip_stress);
788 load_klass(r0, r0);
789 orr(r0, r0, 1);
790 bind(skip_stress);
791 }
792 #endif
793 bind(skip);
794 // Check above kills sender esp in rscratch2. Reload it.
795 ldr(rscratch2, Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
796 }
797
798 // remove frame anchor
799 leave();
800
801 JFR_ONLY(leave_jfr_critical_section();)
802
803 // restore sender esp
804 mov(esp, rscratch2);
805
806 // If we're returning to interpreted code we will shortly be
807 // adjusting SP to allow some space for ESP. If we're returning to
808 // compiled code the saved sender SP was saved in sender_sp, so this
809 // restores it.
810 andr(sp, esp, -16);
811 }
812
813 #if INCLUDE_JFR
814 void InterpreterMacroAssembler::enter_jfr_critical_section() {
815 const Address sampling_critical_section(rthread, in_bytes(SAMPLING_CRITICAL_SECTION_OFFSET_JFR));
816 mov(rscratch1, true);
817 strb(rscratch1, sampling_critical_section);
1055 }
1056
1057
1058 void InterpreterMacroAssembler::profile_taken_branch(Register mdp) {
1059 if (ProfileInterpreter) {
1060 Label profile_continue;
1061
1062 // If no method data exists, go to profile_continue.
1063 test_method_data_pointer(mdp, profile_continue);
1064
1065 // We are taking a branch. Increment the taken count.
1066 increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
1067
1068 // The method data pointer needs to be updated to reflect the new target.
1069 update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
1070 bind(profile_continue);
1071 }
1072 }
1073
1074
1075 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp, bool acmp) {
1076 if (ProfileInterpreter) {
1077 Label profile_continue;
1078
1079 // If no method data exists, go to profile_continue.
1080 test_method_data_pointer(mdp, profile_continue);
1081
1082 // We are not taking a branch. Increment the not taken count.
1083 increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
1084
1085 // The method data pointer needs to be updated to correspond to
1086 // the next bytecode
1087 update_mdp_by_constant(mdp, acmp ? in_bytes(ACmpData::acmp_data_size()) : in_bytes(BranchData::branch_data_size()));
1088 bind(profile_continue);
1089 }
1090 }
1091
1092
1093 void InterpreterMacroAssembler::profile_call(Register mdp) {
1094 if (ProfileInterpreter) {
1095 Label profile_continue;
1096
1097 // If no method data exists, go to profile_continue.
1098 test_method_data_pointer(mdp, profile_continue);
1099
1100 // We are making a call. Increment the count.
1101 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1102
1103 // The method data pointer needs to be updated to reflect the new target.
1104 update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1105 bind(profile_continue);
1106 }
1107 }
1390 // case_array_offset_in_bytes()
1391 movw(reg2, in_bytes(MultiBranchData::per_case_size()));
1392 movw(rscratch1, in_bytes(MultiBranchData::case_array_offset()));
1393 Assembler::maddw(index, index, reg2, rscratch1);
1394
1395 // Update the case count
1396 increment_mdp_data_at(mdp,
1397 index,
1398 in_bytes(MultiBranchData::relative_count_offset()));
1399
1400 // The method data pointer needs to be updated.
1401 update_mdp_by_offset(mdp,
1402 index,
1403 in_bytes(MultiBranchData::
1404 relative_displacement_offset()));
1405
1406 bind(profile_continue);
1407 }
1408 }
1409
1410 template <class ArrayData> void InterpreterMacroAssembler::profile_array_type(Register mdp,
1411 Register array,
1412 Register tmp) {
1413 if (ProfileInterpreter) {
1414 Label profile_continue;
1415
1416 // If no method data exists, go to profile_continue.
1417 test_method_data_pointer(mdp, profile_continue);
1418
1419 mov(tmp, array);
1420 profile_obj_type(tmp, Address(mdp, in_bytes(ArrayData::array_offset())));
1421
1422 Label not_flat;
1423 test_non_flat_array_oop(array, tmp, not_flat);
1424
1425 set_mdp_flag_at(mdp, ArrayData::flat_array_byte_constant());
1426
1427 bind(not_flat);
1428
1429 Label not_null_free;
1430 test_non_null_free_array_oop(array, tmp, not_null_free);
1431
1432 set_mdp_flag_at(mdp, ArrayData::null_free_array_byte_constant());
1433
1434 bind(not_null_free);
1435
1436 bind(profile_continue);
1437 }
1438 }
1439
1440 template void InterpreterMacroAssembler::profile_array_type<ArrayLoadData>(Register mdp,
1441 Register array,
1442 Register tmp);
1443 template void InterpreterMacroAssembler::profile_array_type<ArrayStoreData>(Register mdp,
1444 Register array,
1445 Register tmp);
1446
1447 void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2) {
1448 if (ProfileInterpreter) {
1449 Label profile_continue;
1450
1451 // If no method data exists, go to profile_continue.
1452 test_method_data_pointer(mdp, profile_continue);
1453
1454 Label done, update;
1455 cbnz(element, update);
1456 set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
1457 b(done);
1458
1459 bind(update);
1460 load_klass(tmp, element);
1461
1462 // Record the object type.
1463 record_klass_in_profile(tmp, mdp, tmp2);
1464
1465 bind(done);
1466
1467 // The method data pointer needs to be updated.
1468 update_mdp_by_constant(mdp, in_bytes(ArrayStoreData::array_store_data_size()));
1469
1470 bind(profile_continue);
1471 }
1472 }
1473
1474
1475 void InterpreterMacroAssembler::profile_element_type(Register mdp,
1476 Register element,
1477 Register tmp) {
1478 if (ProfileInterpreter) {
1479 Label profile_continue;
1480
1481 // If no method data exists, go to profile_continue.
1482 test_method_data_pointer(mdp, profile_continue);
1483
1484 mov(tmp, element);
1485 profile_obj_type(tmp, Address(mdp, in_bytes(ArrayLoadData::element_offset())));
1486
1487 // The method data pointer needs to be updated.
1488 update_mdp_by_constant(mdp, in_bytes(ArrayLoadData::array_load_data_size()));
1489
1490 bind(profile_continue);
1491 }
1492 }
1493
1494 void InterpreterMacroAssembler::profile_acmp(Register mdp,
1495 Register left,
1496 Register right,
1497 Register tmp) {
1498 if (ProfileInterpreter) {
1499 Label profile_continue;
1500
1501 // If no method data exists, go to profile_continue.
1502 test_method_data_pointer(mdp, profile_continue);
1503
1504 mov(tmp, left);
1505 profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::left_offset())));
1506
1507 Label left_not_inline_type;
1508 test_oop_is_not_inline_type(left, tmp, left_not_inline_type);
1509 set_mdp_flag_at(mdp, ACmpData::left_inline_type_byte_constant());
1510 bind(left_not_inline_type);
1511
1512 mov(tmp, right);
1513 profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::right_offset())));
1514
1515 Label right_not_inline_type;
1516 test_oop_is_not_inline_type(right, tmp, right_not_inline_type);
1517 set_mdp_flag_at(mdp, ACmpData::right_inline_type_byte_constant());
1518 bind(right_not_inline_type);
1519
1520 bind(profile_continue);
1521 }
1522 }
1523
1524 void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) {
1525 if (state == atos) {
1526 MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line);
1527 }
1528 }
1529
1530 void InterpreterMacroAssembler::notify_method_entry() {
1531 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1532 // track stack depth. If it is possible to enter interp_only_mode we add
1533 // the code to check if the event should be sent.
1534 if (JvmtiExport::can_post_interpreter_events()) {
1535 Label L;
1536 ldrw(r3, Address(rthread, JavaThread::interp_only_mode_offset()));
1537 cbzw(r3, L);
1538 call_VM(noreg, CAST_FROM_FN_PTR(address,
1539 InterpreterRuntime::post_method_entry));
1540 bind(L);
1541 }
1542
1543 if (DTraceMethodProbes) {
1867 profile_obj_type(tmp, mdo_arg_addr);
1868
1869 int to_add = in_bytes(TypeStackSlotEntries::per_arg_size());
1870 off_to_args += to_add;
1871 }
1872
1873 if (MethodData::profile_return()) {
1874 ldr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())));
1875 sub(tmp, tmp, TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count());
1876 }
1877
1878 add(rscratch1, mdp, off_to_args);
1879 bind(done);
1880 mov(mdp, rscratch1);
1881
1882 if (MethodData::profile_return()) {
1883 // We're right after the type profile for the last
1884 // argument. tmp is the number of cells left in the
1885 // CallTypeData/VirtualCallTypeData to reach its end. Non null
1886 // if there's a return to profile.
1887 assert(SingleTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
1888 add(mdp, mdp, tmp, LSL, exact_log2(DataLayout::cell_size));
1889 }
1890 str(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1891 } else {
1892 assert(MethodData::profile_return(), "either profile call args or call ret");
1893 update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size()));
1894 }
1895
1896 // mdp points right after the end of the
1897 // CallTypeData/VirtualCallTypeData, right after the cells for the
1898 // return value type if there's one
1899
1900 bind(profile_continue);
1901 }
1902 }
1903
1904 void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) {
1905 assert_different_registers(mdp, ret, tmp, rbcp);
1906 if (ProfileInterpreter && MethodData::profile_return()) {
1907 Label profile_continue, done;
1913
1914 // If we don't profile all invoke bytecodes we must make sure
1915 // it's a bytecode we indeed profile. We can't go back to the
1916 // beginning of the ProfileData we intend to update to check its
1917 // type because we're right after it and we don't known its
1918 // length
1919 Label do_profile;
1920 ldrb(rscratch1, Address(rbcp, 0));
1921 cmp(rscratch1, (u1)Bytecodes::_invokedynamic);
1922 br(Assembler::EQ, do_profile);
1923 cmp(rscratch1, (u1)Bytecodes::_invokehandle);
1924 br(Assembler::EQ, do_profile);
1925 get_method(tmp);
1926 ldrh(rscratch1, Address(tmp, Method::intrinsic_id_offset()));
1927 subs(zr, rscratch1, static_cast<int>(vmIntrinsics::_compiledLambdaForm));
1928 br(Assembler::NE, profile_continue);
1929
1930 bind(do_profile);
1931 }
1932
1933 Address mdo_ret_addr(mdp, -in_bytes(SingleTypeEntry::size()));
1934 mov(tmp, ret);
1935 profile_obj_type(tmp, mdo_ret_addr);
1936
1937 bind(profile_continue);
1938 }
1939 }
1940
1941 void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2) {
1942 assert_different_registers(rscratch1, rscratch2, mdp, tmp1, tmp2);
1943 if (ProfileInterpreter && MethodData::profile_parameters()) {
1944 Label profile_continue, done;
1945
1946 test_method_data_pointer(mdp, profile_continue);
1947
1948 // Load the offset of the area within the MDO used for
1949 // parameters. If it's negative we're not profiling any parameters
1950 ldrw(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset())));
1951 tbnz(tmp1, 31, profile_continue); // i.e. sign bit set
1952
1953 // Compute a pointer to the area for parameters from the offset
|