15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "compiler/compiler_globals.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/barrierSetAssembler.hpp"
30 #include "interp_masm_aarch64.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interpreterRuntime.hpp"
33 #include "logging/log.hpp"
34 #include "oops/arrayOop.hpp"
35 #include "oops/markWord.hpp"
36 #include "oops/method.hpp"
37 #include "oops/methodData.hpp"
38 #include "oops/resolvedFieldEntry.hpp"
39 #include "oops/resolvedIndyEntry.hpp"
40 #include "oops/resolvedMethodEntry.hpp"
41 #include "prims/jvmtiExport.hpp"
42 #include "prims/jvmtiThreadState.hpp"
43 #include "runtime/basicLock.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/javaThread.hpp"
46 #include "runtime/safepointMechanism.hpp"
47 #include "runtime/sharedRuntime.hpp"
48 #include "utilities/powerOfTwo.hpp"
49
50 void InterpreterMacroAssembler::narrow(Register result) {
51
52 // Get method->_constMethod->_result_type
53 ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
54 ldr(rscratch1, Address(rscratch1, Method::const_offset()));
55 ldrb(rscratch1, Address(rscratch1, ConstMethod::result_type_offset()));
56
57 Label done, notBool, notByte, notChar;
191 ldrw(index, Address(rbcp, bcp_offset));
192 } else if (index_size == sizeof(u1)) {
193 load_unsigned_byte(index, Address(rbcp, bcp_offset));
194 } else {
195 ShouldNotReachHere();
196 }
197 }
198
199 void InterpreterMacroAssembler::get_method_counters(Register method,
200 Register mcs, Label& skip) {
201 Label has_counters;
202 ldr(mcs, Address(method, Method::method_counters_offset()));
203 cbnz(mcs, has_counters);
204 call_VM(noreg, CAST_FROM_FN_PTR(address,
205 InterpreterRuntime::build_method_counters), method);
206 ldr(mcs, Address(method, Method::method_counters_offset()));
207 cbz(mcs, skip); // No MethodCounters allocated, OutOfMemory
208 bind(has_counters);
209 }
210
211 // Load object from cpool->resolved_references(index)
212 void InterpreterMacroAssembler::load_resolved_reference_at_index(
213 Register result, Register index, Register tmp) {
214 assert_different_registers(result, index);
215
216 get_constant_pool(result);
217 // load pointer for resolved_references[] objArray
218 ldr(result, Address(result, ConstantPool::cache_offset()));
219 ldr(result, Address(result, ConstantPoolCache::resolved_references_offset()));
220 resolve_oop_handle(result, tmp, rscratch2);
221 // Add in the index
222 add(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
223 load_heap_oop(result, Address(result, index, Address::uxtw(LogBytesPerHeapOop)), tmp, rscratch2);
224 }
225
226 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
227 Register cpool, Register index, Register klass, Register temp) {
228 add(temp, cpool, index, LSL, LogBytesPerWord);
229 ldrh(temp, Address(temp, sizeof(ConstantPool))); // temp = resolved_klass_index
230 ldr(klass, Address(cpool, ConstantPool::resolved_klasses_offset())); // klass = cpool->_resolved_klasses
231 add(klass, klass, temp, LSL, LogBytesPerWord);
232 ldr(klass, Address(klass, Array<Klass*>::base_offset_in_bytes()));
233 }
234
235 // Generate a subtype check: branch to ok_is_subtype if sub_klass is a
236 // subtype of super_klass.
237 //
238 // Args:
239 // r0: superklass
240 // Rsub_klass: subklass
241 //
242 // Kills:
243 // r2, r5
244 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
245 Label& ok_is_subtype) {
246 assert(Rsub_klass != r0, "r0 holds superklass");
247 assert(Rsub_klass != r2, "r2 holds 2ndary super array length");
248 assert(Rsub_klass != r5, "r5 holds 2ndary super array scan ptr");
249
250 // Profile the not-null value's klass.
251 profile_typecheck(r2, Rsub_klass, r5); // blows r2, reloads r5
252
253 // Do the check.
254 check_klass_subtype(Rsub_klass, r0, r2, ok_is_subtype); // blows r2
255 }
256
257 // Java Expression Stack
258
259 void InterpreterMacroAssembler::pop_ptr(Register r) {
260 ldr(r, post(esp, wordSize));
261 }
262
263 void InterpreterMacroAssembler::pop_i(Register r) {
264 ldrw(r, post(esp, wordSize));
265 }
266
267 void InterpreterMacroAssembler::pop_l(Register r) {
268 ldr(r, post(esp, 2 * Interpreter::stackElementSize));
269 }
270
271 void InterpreterMacroAssembler::push_ptr(Register r) {
607 br(Assembler::AL, fast_path);
608 bind(slow_path);
609 push(state);
610 set_last_Java_frame(esp, rfp, pc(), rscratch1);
611 super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), rthread);
612 reset_last_Java_frame(true);
613 pop(state);
614 bind(fast_path);
615
616 // JVMTI support. Make sure the safepoint poll test is issued prior.
617 if (notify_jvmdi) {
618 notify_method_exit(state, NotifyJVMTI); // preserve TOSCA
619 } else {
620 notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
621 }
622
623 // remove activation
624 // get sender esp
625 ldr(rscratch2,
626 Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
627 if (StackReservedPages > 0) {
628 // testing if reserved zone needs to be re-enabled
629 Label no_reserved_zone_enabling;
630
631 // check if already enabled - if so no re-enabling needed
632 assert(sizeof(StackOverflow::StackGuardState) == 4, "unexpected size");
633 ldrw(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
634 cmpw(rscratch1, (u1)StackOverflow::stack_guard_enabled);
635 br(Assembler::EQ, no_reserved_zone_enabling);
636
637 // look for an overflow into the stack reserved zone, i.e.
638 // interpreter_frame_sender_sp <= JavaThread::reserved_stack_activation
639 ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset()));
640 cmp(rscratch2, rscratch1);
641 br(Assembler::LS, no_reserved_zone_enabling);
642
643 JFR_ONLY(leave_jfr_critical_section();)
644
645 call_VM_leaf(
646 CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), rthread);
647 call_VM(noreg, CAST_FROM_FN_PTR(address,
648 InterpreterRuntime::throw_delayed_StackOverflowError));
649 should_not_reach_here();
650
651 bind(no_reserved_zone_enabling);
652 }
653
654 // remove frame anchor
655 leave();
656
657 JFR_ONLY(leave_jfr_critical_section();)
658
659 // restore sender esp
660 mov(esp, rscratch2);
661
662 // If we're returning to interpreted code we will shortly be
663 // adjusting SP to allow some space for ESP. If we're returning to
664 // compiled code the saved sender SP was saved in sender_sp, so this
665 // restores it.
666 andr(sp, esp, -16);
667 }
668
669 #if INCLUDE_JFR
670 void InterpreterMacroAssembler::enter_jfr_critical_section() {
671 const Address sampling_critical_section(rthread, in_bytes(SAMPLING_CRITICAL_SECTION_OFFSET_JFR));
672 mov(rscratch1, true);
673 strb(rscratch1, sampling_critical_section);
911 }
912
913
914 void InterpreterMacroAssembler::profile_taken_branch(Register mdp) {
915 if (ProfileInterpreter) {
916 Label profile_continue;
917
918 // If no method data exists, go to profile_continue.
919 test_method_data_pointer(mdp, profile_continue);
920
921 // We are taking a branch. Increment the taken count.
922 increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
923
924 // The method data pointer needs to be updated to reflect the new target.
925 update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
926 bind(profile_continue);
927 }
928 }
929
930
931 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
932 if (ProfileInterpreter) {
933 Label profile_continue;
934
935 // If no method data exists, go to profile_continue.
936 test_method_data_pointer(mdp, profile_continue);
937
938 // We are not taking a branch. Increment the not taken count.
939 increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
940
941 // The method data pointer needs to be updated to correspond to
942 // the next bytecode
943 update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));
944 bind(profile_continue);
945 }
946 }
947
948
949 void InterpreterMacroAssembler::profile_call(Register mdp) {
950 if (ProfileInterpreter) {
951 Label profile_continue;
952
953 // If no method data exists, go to profile_continue.
954 test_method_data_pointer(mdp, profile_continue);
955
956 // We are making a call. Increment the count.
957 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
958
959 // The method data pointer needs to be updated to reflect the new target.
960 update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
961 bind(profile_continue);
962 }
963 }
1246 // case_array_offset_in_bytes()
1247 movw(reg2, in_bytes(MultiBranchData::per_case_size()));
1248 movw(rscratch1, in_bytes(MultiBranchData::case_array_offset()));
1249 Assembler::maddw(index, index, reg2, rscratch1);
1250
1251 // Update the case count
1252 increment_mdp_data_at(mdp,
1253 index,
1254 in_bytes(MultiBranchData::relative_count_offset()));
1255
1256 // The method data pointer needs to be updated.
1257 update_mdp_by_offset(mdp,
1258 index,
1259 in_bytes(MultiBranchData::
1260 relative_displacement_offset()));
1261
1262 bind(profile_continue);
1263 }
1264 }
1265
1266 void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) {
1267 if (state == atos) {
1268 MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line);
1269 }
1270 }
1271
1272 void InterpreterMacroAssembler::notify_method_entry() {
1273 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1274 // track stack depth. If it is possible to enter interp_only_mode we add
1275 // the code to check if the event should be sent.
1276 if (JvmtiExport::can_post_interpreter_events()) {
1277 Label L;
1278 ldrw(r3, Address(rthread, JavaThread::interp_only_mode_offset()));
1279 cbzw(r3, L);
1280 call_VM(noreg, CAST_FROM_FN_PTR(address,
1281 InterpreterRuntime::post_method_entry));
1282 bind(L);
1283 }
1284
1285 if (DTraceMethodProbes) {
1544 profile_obj_type(tmp, mdo_arg_addr);
1545
1546 int to_add = in_bytes(TypeStackSlotEntries::per_arg_size());
1547 off_to_args += to_add;
1548 }
1549
1550 if (MethodData::profile_return()) {
1551 ldr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())));
1552 sub(tmp, tmp, TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count());
1553 }
1554
1555 add(rscratch1, mdp, off_to_args);
1556 bind(done);
1557 mov(mdp, rscratch1);
1558
1559 if (MethodData::profile_return()) {
1560 // We're right after the type profile for the last
1561 // argument. tmp is the number of cells left in the
1562 // CallTypeData/VirtualCallTypeData to reach its end. Non null
1563 // if there's a return to profile.
1564 assert(ReturnTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
1565 add(mdp, mdp, tmp, LSL, exact_log2(DataLayout::cell_size));
1566 }
1567 str(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1568 } else {
1569 assert(MethodData::profile_return(), "either profile call args or call ret");
1570 update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size()));
1571 }
1572
1573 // mdp points right after the end of the
1574 // CallTypeData/VirtualCallTypeData, right after the cells for the
1575 // return value type if there's one
1576
1577 bind(profile_continue);
1578 }
1579 }
1580
1581 void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) {
1582 assert_different_registers(mdp, ret, tmp, rbcp);
1583 if (ProfileInterpreter && MethodData::profile_return()) {
1584 Label profile_continue, done;
1590
1591 // If we don't profile all invoke bytecodes we must make sure
1592 // it's a bytecode we indeed profile. We can't go back to the
1593 // beginning of the ProfileData we intend to update to check its
1594 // type because we're right after it and we don't known its
1595 // length
1596 Label do_profile;
1597 ldrb(rscratch1, Address(rbcp, 0));
1598 cmp(rscratch1, (u1)Bytecodes::_invokedynamic);
1599 br(Assembler::EQ, do_profile);
1600 cmp(rscratch1, (u1)Bytecodes::_invokehandle);
1601 br(Assembler::EQ, do_profile);
1602 get_method(tmp);
1603 ldrh(rscratch1, Address(tmp, Method::intrinsic_id_offset()));
1604 subs(zr, rscratch1, static_cast<int>(vmIntrinsics::_compiledLambdaForm));
1605 br(Assembler::NE, profile_continue);
1606
1607 bind(do_profile);
1608 }
1609
1610 Address mdo_ret_addr(mdp, -in_bytes(ReturnTypeEntry::size()));
1611 mov(tmp, ret);
1612 profile_obj_type(tmp, mdo_ret_addr);
1613
1614 bind(profile_continue);
1615 }
1616 }
1617
1618 void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2) {
1619 assert_different_registers(rscratch1, rscratch2, mdp, tmp1, tmp2);
1620 if (ProfileInterpreter && MethodData::profile_parameters()) {
1621 Label profile_continue, done;
1622
1623 test_method_data_pointer(mdp, profile_continue);
1624
1625 // Load the offset of the area within the MDO used for
1626 // parameters. If it's negative we're not profiling any parameters
1627 ldrw(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset())));
1628 tbnz(tmp1, 31, profile_continue); // i.e. sign bit set
1629
1630 // Compute a pointer to the area for parameters from the offset
|
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "compiler/compiler_globals.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/barrierSetAssembler.hpp"
30 #include "interp_masm_aarch64.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interpreterRuntime.hpp"
33 #include "logging/log.hpp"
34 #include "oops/arrayOop.hpp"
35 #include "oops/constMethodFlags.hpp"
36 #include "oops/markWord.hpp"
37 #include "oops/method.hpp"
38 #include "oops/methodData.hpp"
39 #include "oops/inlineKlass.hpp"
40 #include "oops/resolvedFieldEntry.hpp"
41 #include "oops/resolvedIndyEntry.hpp"
42 #include "oops/resolvedMethodEntry.hpp"
43 #include "prims/jvmtiExport.hpp"
44 #include "prims/jvmtiThreadState.hpp"
45 #include "runtime/basicLock.hpp"
46 #include "runtime/frame.inline.hpp"
47 #include "runtime/javaThread.hpp"
48 #include "runtime/safepointMechanism.hpp"
49 #include "runtime/sharedRuntime.hpp"
50 #include "utilities/powerOfTwo.hpp"
51
52 void InterpreterMacroAssembler::narrow(Register result) {
53
54 // Get method->_constMethod->_result_type
55 ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
56 ldr(rscratch1, Address(rscratch1, Method::const_offset()));
57 ldrb(rscratch1, Address(rscratch1, ConstMethod::result_type_offset()));
58
59 Label done, notBool, notByte, notChar;
193 ldrw(index, Address(rbcp, bcp_offset));
194 } else if (index_size == sizeof(u1)) {
195 load_unsigned_byte(index, Address(rbcp, bcp_offset));
196 } else {
197 ShouldNotReachHere();
198 }
199 }
200
201 void InterpreterMacroAssembler::get_method_counters(Register method,
202 Register mcs, Label& skip) {
203 Label has_counters;
204 ldr(mcs, Address(method, Method::method_counters_offset()));
205 cbnz(mcs, has_counters);
206 call_VM(noreg, CAST_FROM_FN_PTR(address,
207 InterpreterRuntime::build_method_counters), method);
208 ldr(mcs, Address(method, Method::method_counters_offset()));
209 cbz(mcs, skip); // No MethodCounters allocated, OutOfMemory
210 bind(has_counters);
211 }
212
213 void InterpreterMacroAssembler::allocate_instance(Register klass, Register new_obj,
214 Register t1, Register t2,
215 bool clear_fields, Label& alloc_failed) {
216 MacroAssembler::allocate_instance(klass, new_obj, t1, t2, clear_fields, alloc_failed);
217 if (DTraceAllocProbes) {
218 // Trigger dtrace event for fastpath
219 push(atos);
220 call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), new_obj);
221 pop(atos);
222 }
223 }
224
225 void InterpreterMacroAssembler::read_flat_field(Register entry,
226 Register field_index, Register field_offset,
227 Register temp, Register obj) {
228 Label failed_alloc, slow_path, done;
229 const Register src = field_offset;
230 const Register alloc_temp = r10;
231 const Register dst_temp = field_index;
232 const Register layout_info = temp;
233 assert_different_registers(obj, entry, field_index, field_offset, temp, alloc_temp, rscratch1);
234
235 load_unsigned_byte(temp, Address(entry, in_bytes(ResolvedFieldEntry::flags_offset())));
236 // If the field is nullable, jump to slow path
237 tbz(temp, ResolvedFieldEntry::is_null_free_inline_type_shift, slow_path);
238
239 // Grab the inline field klass
240 ldr(rscratch1, Address(entry, in_bytes(ResolvedFieldEntry::field_holder_offset())));
241 inline_layout_info(rscratch1, field_index, layout_info);
242
243 const Register field_klass = dst_temp;
244 ldr(field_klass, Address(layout_info, in_bytes(InlineLayoutInfo::klass_offset())));
245
246 // allocate buffer
247 push(obj); // save holder
248 allocate_instance(field_klass, obj, alloc_temp, rscratch2, false, failed_alloc);
249
250 // Have an oop instance buffer, copy into it
251 payload_address(obj, dst_temp, field_klass); // danger, uses rscratch1
252 pop(alloc_temp); // restore holder
253 lea(src, Address(alloc_temp, field_offset));
254 // call_VM_leaf, clobbers a few regs, save restore new obj
255 push(obj);
256 flat_field_copy(IS_DEST_UNINITIALIZED, src, dst_temp, layout_info);
257 pop(obj);
258 b(done);
259
260 bind(failed_alloc);
261 pop(obj);
262 bind(slow_path);
263 call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flat_field),
264 obj, entry);
265
266 bind(done);
267 membar(Assembler::StoreStore);
268 }
269
270 void InterpreterMacroAssembler::write_flat_field(Register entry, Register field_offset,
271 Register tmp1, Register tmp2,
272 Register obj) {
273 assert_different_registers(entry, field_offset, tmp1, tmp2, obj);
274 Label slow_path, done;
275
276 load_unsigned_byte(tmp1, Address(entry, in_bytes(ResolvedFieldEntry::flags_offset())));
277 test_field_is_not_null_free_inline_type(tmp1, noreg /* temp */, slow_path);
278
279 null_check(r0); // FIXME JDK-8341120
280
281 add(obj, obj, field_offset);
282
283 load_klass(tmp1, r0);
284 payload_address(r0, r0, tmp1);
285
286 Register layout_info = field_offset;
287 load_unsigned_short(tmp1, Address(entry, in_bytes(ResolvedFieldEntry::field_index_offset())));
288 ldr(tmp2, Address(entry, in_bytes(ResolvedFieldEntry::field_holder_offset())));
289 inline_layout_info(tmp2, tmp1, layout_info);
290
291 flat_field_copy(IN_HEAP, r0, obj, layout_info);
292 b(done);
293
294 bind(slow_path);
295 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_flat_field), obj, r0, entry);
296 bind(done);
297 }
298
299 // Load object from cpool->resolved_references(index)
300 void InterpreterMacroAssembler::load_resolved_reference_at_index(
301 Register result, Register index, Register tmp) {
302 assert_different_registers(result, index);
303
304 get_constant_pool(result);
305 // load pointer for resolved_references[] objArray
306 ldr(result, Address(result, ConstantPool::cache_offset()));
307 ldr(result, Address(result, ConstantPoolCache::resolved_references_offset()));
308 resolve_oop_handle(result, tmp, rscratch2);
309 // Add in the index
310 add(index, index, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop);
311 load_heap_oop(result, Address(result, index, Address::uxtw(LogBytesPerHeapOop)), tmp, rscratch2);
312 }
313
314 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
315 Register cpool, Register index, Register klass, Register temp) {
316 add(temp, cpool, index, LSL, LogBytesPerWord);
317 ldrh(temp, Address(temp, sizeof(ConstantPool))); // temp = resolved_klass_index
318 ldr(klass, Address(cpool, ConstantPool::resolved_klasses_offset())); // klass = cpool->_resolved_klasses
319 add(klass, klass, temp, LSL, LogBytesPerWord);
320 ldr(klass, Address(klass, Array<Klass*>::base_offset_in_bytes()));
321 }
322
323 // Generate a subtype check: branch to ok_is_subtype if sub_klass is a
324 // subtype of super_klass.
325 //
326 // Args:
327 // r0: superklass
328 // Rsub_klass: subklass
329 //
330 // Kills:
331 // r2, r5
332 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
333 Label& ok_is_subtype,
334 bool profile) {
335 assert(Rsub_klass != r0, "r0 holds superklass");
336 assert(Rsub_klass != r2, "r2 holds 2ndary super array length");
337 assert(Rsub_klass != r5, "r5 holds 2ndary super array scan ptr");
338
339 // Profile the not-null value's klass.
340 if (profile) {
341 profile_typecheck(r2, Rsub_klass, r5); // blows r2, reloads r5
342 }
343
344 // Do the check.
345 check_klass_subtype(Rsub_klass, r0, r2, ok_is_subtype); // blows r2
346 }
347
348 // Java Expression Stack
349
350 void InterpreterMacroAssembler::pop_ptr(Register r) {
351 ldr(r, post(esp, wordSize));
352 }
353
354 void InterpreterMacroAssembler::pop_i(Register r) {
355 ldrw(r, post(esp, wordSize));
356 }
357
358 void InterpreterMacroAssembler::pop_l(Register r) {
359 ldr(r, post(esp, 2 * Interpreter::stackElementSize));
360 }
361
362 void InterpreterMacroAssembler::push_ptr(Register r) {
698 br(Assembler::AL, fast_path);
699 bind(slow_path);
700 push(state);
701 set_last_Java_frame(esp, rfp, pc(), rscratch1);
702 super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), rthread);
703 reset_last_Java_frame(true);
704 pop(state);
705 bind(fast_path);
706
707 // JVMTI support. Make sure the safepoint poll test is issued prior.
708 if (notify_jvmdi) {
709 notify_method_exit(state, NotifyJVMTI); // preserve TOSCA
710 } else {
711 notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
712 }
713
714 // remove activation
715 // get sender esp
716 ldr(rscratch2,
717 Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
718
719 if (StackReservedPages > 0) {
720 // testing if reserved zone needs to be re-enabled
721 Label no_reserved_zone_enabling;
722
723 // check if already enabled - if so no re-enabling needed
724 assert(sizeof(StackOverflow::StackGuardState) == 4, "unexpected size");
725 ldrw(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
726 cmpw(rscratch1, (u1)StackOverflow::stack_guard_enabled);
727 br(Assembler::EQ, no_reserved_zone_enabling);
728
729 // look for an overflow into the stack reserved zone, i.e.
730 // interpreter_frame_sender_sp <= JavaThread::reserved_stack_activation
731 ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset()));
732 cmp(rscratch2, rscratch1);
733 br(Assembler::LS, no_reserved_zone_enabling);
734
735 JFR_ONLY(leave_jfr_critical_section();)
736
737 call_VM_leaf(
738 CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), rthread);
739 call_VM(noreg, CAST_FROM_FN_PTR(address,
740 InterpreterRuntime::throw_delayed_StackOverflowError));
741 should_not_reach_here();
742
743 bind(no_reserved_zone_enabling);
744 }
745
746 if (state == atos && InlineTypeReturnedAsFields) {
747 Label skip;
748 Label not_null;
749 cbnz(r0, not_null);
750 // Returned value is null, zero all return registers because they may belong to oop fields
751 mov(j_rarg1, zr);
752 mov(j_rarg2, zr);
753 mov(j_rarg3, zr);
754 mov(j_rarg4, zr);
755 mov(j_rarg5, zr);
756 mov(j_rarg6, zr);
757 mov(j_rarg7, zr);
758 b(skip);
759 bind(not_null);
760
761 // Check if we are returning an non-null inline type and load its fields into registers
762 test_oop_is_not_inline_type(r0, rscratch2, skip, /* can_be_null= */ false);
763
764 // Load fields from a buffered value with an inline class specific handler
765 load_klass(rscratch1 /*dst*/, r0 /*src*/);
766 ldr(rscratch1, Address(rscratch1, InstanceKlass::adr_inlineklass_fixed_block_offset()));
767 ldr(rscratch1, Address(rscratch1, InlineKlass::unpack_handler_offset()));
768 // Unpack handler can be null if inline type is not scalarizable in returns
769 cbz(rscratch1, skip);
770
771 blr(rscratch1);
772 #ifdef ASSERT
773 // TODO 8284443 Enable
774 if (StressCallingConvention && false) {
775 Label skip_stress;
776 ldr(rscratch1, Address(rfp, frame::interpreter_frame_method_offset * wordSize));
777 ldrw(rscratch1, Address(rscratch1, Method::flags_offset()));
778 tstw(rscratch1, MethodFlags::has_scalarized_return_flag());
779 br(Assembler::EQ, skip_stress);
780 load_klass(r0, r0);
781 orr(r0, r0, 1);
782 bind(skip_stress);
783 }
784 #endif
785 bind(skip);
786 // Check above kills sender esp in rscratch2. Reload it.
787 ldr(rscratch2, Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
788 }
789
790 // remove frame anchor
791 leave();
792
793 JFR_ONLY(leave_jfr_critical_section();)
794
795 // restore sender esp
796 mov(esp, rscratch2);
797
798 // If we're returning to interpreted code we will shortly be
799 // adjusting SP to allow some space for ESP. If we're returning to
800 // compiled code the saved sender SP was saved in sender_sp, so this
801 // restores it.
802 andr(sp, esp, -16);
803 }
804
805 #if INCLUDE_JFR
806 void InterpreterMacroAssembler::enter_jfr_critical_section() {
807 const Address sampling_critical_section(rthread, in_bytes(SAMPLING_CRITICAL_SECTION_OFFSET_JFR));
808 mov(rscratch1, true);
809 strb(rscratch1, sampling_critical_section);
1047 }
1048
1049
1050 void InterpreterMacroAssembler::profile_taken_branch(Register mdp) {
1051 if (ProfileInterpreter) {
1052 Label profile_continue;
1053
1054 // If no method data exists, go to profile_continue.
1055 test_method_data_pointer(mdp, profile_continue);
1056
1057 // We are taking a branch. Increment the taken count.
1058 increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()));
1059
1060 // The method data pointer needs to be updated to reflect the new target.
1061 update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()));
1062 bind(profile_continue);
1063 }
1064 }
1065
1066
1067 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp, bool acmp) {
1068 if (ProfileInterpreter) {
1069 Label profile_continue;
1070
1071 // If no method data exists, go to profile_continue.
1072 test_method_data_pointer(mdp, profile_continue);
1073
1074 // We are not taking a branch. Increment the not taken count.
1075 increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
1076
1077 // The method data pointer needs to be updated to correspond to
1078 // the next bytecode
1079 update_mdp_by_constant(mdp, acmp ? in_bytes(ACmpData::acmp_data_size()) : in_bytes(BranchData::branch_data_size()));
1080 bind(profile_continue);
1081 }
1082 }
1083
1084
1085 void InterpreterMacroAssembler::profile_call(Register mdp) {
1086 if (ProfileInterpreter) {
1087 Label profile_continue;
1088
1089 // If no method data exists, go to profile_continue.
1090 test_method_data_pointer(mdp, profile_continue);
1091
1092 // We are making a call. Increment the count.
1093 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
1094
1095 // The method data pointer needs to be updated to reflect the new target.
1096 update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1097 bind(profile_continue);
1098 }
1099 }
1382 // case_array_offset_in_bytes()
1383 movw(reg2, in_bytes(MultiBranchData::per_case_size()));
1384 movw(rscratch1, in_bytes(MultiBranchData::case_array_offset()));
1385 Assembler::maddw(index, index, reg2, rscratch1);
1386
1387 // Update the case count
1388 increment_mdp_data_at(mdp,
1389 index,
1390 in_bytes(MultiBranchData::relative_count_offset()));
1391
1392 // The method data pointer needs to be updated.
1393 update_mdp_by_offset(mdp,
1394 index,
1395 in_bytes(MultiBranchData::
1396 relative_displacement_offset()));
1397
1398 bind(profile_continue);
1399 }
1400 }
1401
1402 template <class ArrayData> void InterpreterMacroAssembler::profile_array_type(Register mdp,
1403 Register array,
1404 Register tmp) {
1405 if (ProfileInterpreter) {
1406 Label profile_continue;
1407
1408 // If no method data exists, go to profile_continue.
1409 test_method_data_pointer(mdp, profile_continue);
1410
1411 mov(tmp, array);
1412 profile_obj_type(tmp, Address(mdp, in_bytes(ArrayData::array_offset())));
1413
1414 Label not_flat;
1415 test_non_flat_array_oop(array, tmp, not_flat);
1416
1417 set_mdp_flag_at(mdp, ArrayData::flat_array_byte_constant());
1418
1419 bind(not_flat);
1420
1421 Label not_null_free;
1422 test_non_null_free_array_oop(array, tmp, not_null_free);
1423
1424 set_mdp_flag_at(mdp, ArrayData::null_free_array_byte_constant());
1425
1426 bind(not_null_free);
1427
1428 bind(profile_continue);
1429 }
1430 }
1431
1432 template void InterpreterMacroAssembler::profile_array_type<ArrayLoadData>(Register mdp,
1433 Register array,
1434 Register tmp);
1435 template void InterpreterMacroAssembler::profile_array_type<ArrayStoreData>(Register mdp,
1436 Register array,
1437 Register tmp);
1438
1439 void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2) {
1440 if (ProfileInterpreter) {
1441 Label profile_continue;
1442
1443 // If no method data exists, go to profile_continue.
1444 test_method_data_pointer(mdp, profile_continue);
1445
1446 Label done, update;
1447 cbnz(element, update);
1448 set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
1449 b(done);
1450
1451 bind(update);
1452 load_klass(tmp, element);
1453
1454 // Record the object type.
1455 record_klass_in_profile(tmp, mdp, tmp2);
1456
1457 bind(done);
1458
1459 // The method data pointer needs to be updated.
1460 update_mdp_by_constant(mdp, in_bytes(ArrayStoreData::array_store_data_size()));
1461
1462 bind(profile_continue);
1463 }
1464 }
1465
1466
1467 void InterpreterMacroAssembler::profile_element_type(Register mdp,
1468 Register element,
1469 Register tmp) {
1470 if (ProfileInterpreter) {
1471 Label profile_continue;
1472
1473 // If no method data exists, go to profile_continue.
1474 test_method_data_pointer(mdp, profile_continue);
1475
1476 mov(tmp, element);
1477 profile_obj_type(tmp, Address(mdp, in_bytes(ArrayLoadData::element_offset())));
1478
1479 // The method data pointer needs to be updated.
1480 update_mdp_by_constant(mdp, in_bytes(ArrayLoadData::array_load_data_size()));
1481
1482 bind(profile_continue);
1483 }
1484 }
1485
1486 void InterpreterMacroAssembler::profile_acmp(Register mdp,
1487 Register left,
1488 Register right,
1489 Register tmp) {
1490 if (ProfileInterpreter) {
1491 Label profile_continue;
1492
1493 // If no method data exists, go to profile_continue.
1494 test_method_data_pointer(mdp, profile_continue);
1495
1496 mov(tmp, left);
1497 profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::left_offset())));
1498
1499 Label left_not_inline_type;
1500 test_oop_is_not_inline_type(left, tmp, left_not_inline_type);
1501 set_mdp_flag_at(mdp, ACmpData::left_inline_type_byte_constant());
1502 bind(left_not_inline_type);
1503
1504 mov(tmp, right);
1505 profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::right_offset())));
1506
1507 Label right_not_inline_type;
1508 test_oop_is_not_inline_type(right, tmp, right_not_inline_type);
1509 set_mdp_flag_at(mdp, ACmpData::right_inline_type_byte_constant());
1510 bind(right_not_inline_type);
1511
1512 bind(profile_continue);
1513 }
1514 }
1515
1516 void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) {
1517 if (state == atos) {
1518 MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line);
1519 }
1520 }
1521
1522 void InterpreterMacroAssembler::notify_method_entry() {
1523 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1524 // track stack depth. If it is possible to enter interp_only_mode we add
1525 // the code to check if the event should be sent.
1526 if (JvmtiExport::can_post_interpreter_events()) {
1527 Label L;
1528 ldrw(r3, Address(rthread, JavaThread::interp_only_mode_offset()));
1529 cbzw(r3, L);
1530 call_VM(noreg, CAST_FROM_FN_PTR(address,
1531 InterpreterRuntime::post_method_entry));
1532 bind(L);
1533 }
1534
1535 if (DTraceMethodProbes) {
1794 profile_obj_type(tmp, mdo_arg_addr);
1795
1796 int to_add = in_bytes(TypeStackSlotEntries::per_arg_size());
1797 off_to_args += to_add;
1798 }
1799
1800 if (MethodData::profile_return()) {
1801 ldr(tmp, Address(mdp, in_bytes(TypeEntriesAtCall::cell_count_offset())));
1802 sub(tmp, tmp, TypeProfileArgsLimit*TypeStackSlotEntries::per_arg_count());
1803 }
1804
1805 add(rscratch1, mdp, off_to_args);
1806 bind(done);
1807 mov(mdp, rscratch1);
1808
1809 if (MethodData::profile_return()) {
1810 // We're right after the type profile for the last
1811 // argument. tmp is the number of cells left in the
1812 // CallTypeData/VirtualCallTypeData to reach its end. Non null
1813 // if there's a return to profile.
1814 assert(SingleTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
1815 add(mdp, mdp, tmp, LSL, exact_log2(DataLayout::cell_size));
1816 }
1817 str(mdp, Address(rfp, frame::interpreter_frame_mdp_offset * wordSize));
1818 } else {
1819 assert(MethodData::profile_return(), "either profile call args or call ret");
1820 update_mdp_by_constant(mdp, in_bytes(TypeEntriesAtCall::return_only_size()));
1821 }
1822
1823 // mdp points right after the end of the
1824 // CallTypeData/VirtualCallTypeData, right after the cells for the
1825 // return value type if there's one
1826
1827 bind(profile_continue);
1828 }
1829 }
1830
1831 void InterpreterMacroAssembler::profile_return_type(Register mdp, Register ret, Register tmp) {
1832 assert_different_registers(mdp, ret, tmp, rbcp);
1833 if (ProfileInterpreter && MethodData::profile_return()) {
1834 Label profile_continue, done;
1840
1841 // If we don't profile all invoke bytecodes we must make sure
1842 // it's a bytecode we indeed profile. We can't go back to the
1843 // beginning of the ProfileData we intend to update to check its
1844 // type because we're right after it and we don't known its
1845 // length
1846 Label do_profile;
1847 ldrb(rscratch1, Address(rbcp, 0));
1848 cmp(rscratch1, (u1)Bytecodes::_invokedynamic);
1849 br(Assembler::EQ, do_profile);
1850 cmp(rscratch1, (u1)Bytecodes::_invokehandle);
1851 br(Assembler::EQ, do_profile);
1852 get_method(tmp);
1853 ldrh(rscratch1, Address(tmp, Method::intrinsic_id_offset()));
1854 subs(zr, rscratch1, static_cast<int>(vmIntrinsics::_compiledLambdaForm));
1855 br(Assembler::NE, profile_continue);
1856
1857 bind(do_profile);
1858 }
1859
1860 Address mdo_ret_addr(mdp, -in_bytes(SingleTypeEntry::size()));
1861 mov(tmp, ret);
1862 profile_obj_type(tmp, mdo_ret_addr);
1863
1864 bind(profile_continue);
1865 }
1866 }
1867
1868 void InterpreterMacroAssembler::profile_parameters_type(Register mdp, Register tmp1, Register tmp2) {
1869 assert_different_registers(rscratch1, rscratch2, mdp, tmp1, tmp2);
1870 if (ProfileInterpreter && MethodData::profile_parameters()) {
1871 Label profile_continue, done;
1872
1873 test_method_data_pointer(mdp, profile_continue);
1874
1875 // Load the offset of the area within the MDO used for
1876 // parameters. If it's negative we're not profiling any parameters
1877 ldrw(tmp1, Address(mdp, in_bytes(MethodData::parameters_type_data_di_offset()) - in_bytes(MethodData::data_offset())));
1878 tbnz(tmp1, 31, profile_continue); // i.e. sign bit set
1879
1880 // Compute a pointer to the area for parameters from the offset
|