< prev index next > src/hotspot/cpu/x86/interp_masm_x86.cpp
Print this page
#include "interp_masm_x86.hpp"
#include "interpreter/interpreter.hpp"
#include "interpreter/interpreterRuntime.hpp"
#include "logging/log.hpp"
#include "oops/arrayOop.hpp"
+ #include "oops/constMethodFlags.hpp"
#include "oops/markWord.hpp"
#include "oops/methodData.hpp"
#include "oops/method.hpp"
+ #include "oops/inlineKlass.hpp"
#include "oops/resolvedFieldEntry.hpp"
#include "oops/resolvedIndyEntry.hpp"
#include "oops/resolvedMethodEntry.hpp"
#include "prims/jvmtiExport.hpp"
#include "prims/jvmtiThreadState.hpp"
if (MethodData::profile_return()) {
// We're right after the type profile for the last
// argument. tmp is the number of cells left in the
// CallTypeData/VirtualCallTypeData to reach its end. Non null
// if there's a return to profile.
! assert(ReturnTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
shll(tmp, log2i_exact((int)DataLayout::cell_size));
addptr(mdp, tmp);
}
movptr(Address(rbp, frame::interpreter_frame_mdp_offset * wordSize), mdp);
} else {
if (MethodData::profile_return()) {
// We're right after the type profile for the last
// argument. tmp is the number of cells left in the
// CallTypeData/VirtualCallTypeData to reach its end. Non null
// if there's a return to profile.
! assert(SingleTypeEntry::static_cell_count() < TypeStackSlotEntries::per_arg_count(), "can't move past ret type");
shll(tmp, log2i_exact((int)DataLayout::cell_size));
addptr(mdp, tmp);
}
movptr(Address(rbp, frame::interpreter_frame_mdp_offset * wordSize), mdp);
} else {
jcc(Assembler::notEqual, profile_continue);
bind(do_profile);
}
! Address mdo_ret_addr(mdp, -in_bytes(ReturnTypeEntry::size()));
mov(tmp, ret);
profile_obj_type(tmp, mdo_ret_addr);
bind(profile_continue);
}
jcc(Assembler::notEqual, profile_continue);
bind(do_profile);
}
! Address mdo_ret_addr(mdp, -in_bytes(SingleTypeEntry::size()));
mov(tmp, ret);
profile_obj_type(tmp, mdo_ret_addr);
bind(profile_continue);
}
// Rsub_klass: subklass
//
// Kills:
// rcx, rdi
void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
! Label& ok_is_subtype) {
assert(Rsub_klass != rax, "rax holds superklass");
LP64_ONLY(assert(Rsub_klass != r14, "r14 holds locals");)
LP64_ONLY(assert(Rsub_klass != r13, "r13 holds bcp");)
assert(Rsub_klass != rcx, "rcx holds 2ndary super array length");
assert(Rsub_klass != rdi, "rdi holds 2ndary super array scan ptr");
// Profile the not-null value's klass.
! profile_typecheck(rcx, Rsub_klass, rdi); // blows rcx, reloads rdi
// Do the check.
check_klass_subtype(Rsub_klass, rax, rcx, ok_is_subtype); // blows rcx
}
// Rsub_klass: subklass
//
// Kills:
// rcx, rdi
void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
! Label& ok_is_subtype,
+ bool profile) {
assert(Rsub_klass != rax, "rax holds superklass");
LP64_ONLY(assert(Rsub_klass != r14, "r14 holds locals");)
LP64_ONLY(assert(Rsub_klass != r13, "r13 holds bcp");)
assert(Rsub_klass != rcx, "rcx holds 2ndary super array length");
assert(Rsub_klass != rdi, "rdi holds 2ndary super array scan ptr");
// Profile the not-null value's klass.
! if (profile) {
+ profile_typecheck(rcx, Rsub_klass, rdi); // blows rcx, reloads rdi
+ }
// Do the check.
check_klass_subtype(Rsub_klass, rax, rcx, ok_is_subtype); // blows rcx
}
const Address do_not_unlock_if_synchronized(rthread,
in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
movbool(rbx, do_not_unlock_if_synchronized);
movbool(do_not_unlock_if_synchronized, false); // reset the flag
! // get method access flags
movptr(rcx, Address(rbp, frame::interpreter_frame_method_offset * wordSize));
movl(rcx, Address(rcx, Method::access_flags_offset()));
testl(rcx, JVM_ACC_SYNCHRONIZED);
jcc(Assembler::zero, unlocked);
const Address do_not_unlock_if_synchronized(rthread,
in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
movbool(rbx, do_not_unlock_if_synchronized);
movbool(do_not_unlock_if_synchronized, false); // reset the flag
! // get method access flags
movptr(rcx, Address(rbp, frame::interpreter_frame_method_offset * wordSize));
movl(rcx, Address(rcx, Method::access_flags_offset()));
testl(rcx, JVM_ACC_SYNCHRONIZED);
jcc(Assembler::zero, unlocked);
notify_method_exit(state, NotifyJVMTI); // preserve TOSCA
} else {
notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
}
- // remove activation
- // get sender sp
- movptr(rbx,
- Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize));
if (StackReservedPages > 0) {
// testing if reserved zone needs to be re-enabled
Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
Label no_reserved_zone_enabling;
NOT_LP64(get_thread(rthread);)
notify_method_exit(state, NotifyJVMTI); // preserve TOSCA
} else {
notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
}
if (StackReservedPages > 0) {
+ movptr(rbx,
+ Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize));
// testing if reserved zone needs to be re-enabled
Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
Label no_reserved_zone_enabling;
NOT_LP64(get_thread(rthread);)
InterpreterRuntime::throw_delayed_StackOverflowError));
should_not_reach_here();
bind(no_reserved_zone_enabling);
}
+
+ // remove activation
+ // get sender sp
+ movptr(rbx,
+ Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize));
+
+ if (state == atos && InlineTypeReturnedAsFields) {
+ // Check if we are returning an non-null inline type and load its fields into registers
+ Label skip;
+ test_oop_is_not_inline_type(rax, rscratch1, skip);
+
+ #ifndef _LP64
+ super_call_VM_leaf(StubRoutines::load_inline_type_fields_in_regs());
+ #else
+ // Load fields from a buffered value with an inline class specific handler
+ load_klass(rdi, rax, rscratch1);
+ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
+ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
+ // Unpack handler can be null if inline type is not scalarizable in returns
+ testptr(rdi, rdi);
+ jcc(Assembler::zero, skip);
+ call(rdi);
+ #endif
+ #ifdef ASSERT
+ // TODO 8284443 Enable
+ if (StressCallingConvention && false) {
+ Label skip_stress;
+ movptr(rscratch1, Address(rbp, frame::interpreter_frame_method_offset * wordSize));
+ movl(rscratch1, Address(rscratch1, Method::flags_offset()));
+ testl(rcx, MethodFlags::has_scalarized_return_flag());
+ jcc(Assembler::zero, skip_stress);
+ load_klass(rax, rax, rscratch1);
+ orptr(rax, 1);
+ bind(skip_stress);
+ }
+ #endif
+ // call above kills the value in rbx. Reload it.
+ movptr(rbx, Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize));
+ bind(skip);
+ }
leave(); // remove frame anchor
pop(ret_addr); // get return address
mov(rsp, rbx); // set sp to sender sp
pop_cont_fastpath();
}
testptr(mcs, mcs);
jcc(Assembler::zero, skip); // No MethodCounters allocated, OutOfMemory
bind(has_counters);
}
+ void InterpreterMacroAssembler::allocate_instance(Register klass, Register new_obj,
+ Register t1, Register t2,
+ bool clear_fields, Label& alloc_failed) {
+ MacroAssembler::allocate_instance(klass, new_obj, t1, t2, clear_fields, alloc_failed);
+ if (DTraceMethodProbes) {
+ // Trigger dtrace event for fastpath
+ push(atos);
+ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), new_obj);
+ pop(atos);
+ }
+ }
+
+ void InterpreterMacroAssembler::read_flat_field(Register entry, Register tmp1, Register tmp2, Register obj) {
+ Label alloc_failed, empty_value, done;
+ const Register alloc_temp = LP64_ONLY(rscratch1) NOT_LP64(rsi);
+ const Register dst_temp = LP64_ONLY(rscratch2) NOT_LP64(rdi);
+ assert_different_registers(obj, entry, tmp1, tmp2, dst_temp, r8, r9);
+
+ // FIXME: code below could be re-written to better use InlineLayoutInfo data structure
+ // see aarch64 version
+
+ // Grap the inline field klass
+ const Register field_klass = tmp1;
+ load_unsigned_short(tmp2, Address(entry, in_bytes(ResolvedFieldEntry::field_index_offset())));
+ movptr(tmp1, Address(entry, ResolvedFieldEntry::field_holder_offset()));
+ get_inline_type_field_klass(tmp1, tmp2, field_klass);
+
+ //check for empty value klass
+ test_klass_is_empty_inline_type(field_klass, dst_temp, empty_value);
+
+ // allocate buffer
+ push(obj); // push object being read from // FIXME spilling on stack could probably be avoided by using tmp2
+ allocate_instance(field_klass, obj, alloc_temp, dst_temp, false, alloc_failed);
+
+ // Have an oop instance buffer, copy into it
+ load_unsigned_short(r9, Address(entry, in_bytes(ResolvedFieldEntry::field_index_offset())));
+ movptr(r8, Address(entry, in_bytes(ResolvedFieldEntry::field_holder_offset())));
+ inline_layout_info(r8, r9, r8); // holder, index, info => InlineLayoutInfo into r8
+
+ data_for_oop(obj, dst_temp, field_klass);
+ pop(alloc_temp); // restore object being read from
+ load_sized_value(tmp2, Address(entry, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
+ lea(tmp2, Address(alloc_temp, tmp2));
+ // call_VM_leaf, clobbers a few regs, save restore new obj
+ push(obj);
+ // access_value_copy(IS_DEST_UNINITIALIZED, tmp2, dst_temp, field_klass);
+ flat_field_copy(IS_DEST_UNINITIALIZED, tmp2, dst_temp, r8);
+ pop(obj);
+ jmp(done);
+
+ bind(empty_value);
+ get_empty_inline_type_oop(field_klass, dst_temp, obj);
+ jmp(done);
+
+ bind(alloc_failed);
+ pop(obj);
+ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_flat_field),
+ obj, entry);
+ get_vm_result(obj, r15_thread);
+ bind(done);
+ }
+
+ void InterpreterMacroAssembler::read_flat_element(Register array, Register index,
+ Register t1, Register t2,
+ Register obj) {
+ assert_different_registers(array, index, t1, t2);
+ Label alloc_failed, empty_value, done;
+ const Register array_klass = t2;
+ const Register elem_klass = t1;
+ const Register alloc_temp = LP64_ONLY(rscratch1) NOT_LP64(rsi);
+ const Register dst_temp = LP64_ONLY(rscratch2) NOT_LP64(rdi);
+
+ // load in array->klass()->element_klass()
+ Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
+ load_klass(array_klass, array, tmp_load_klass);
+ movptr(elem_klass, Address(array_klass, ArrayKlass::element_klass_offset()));
+
+ //check for empty value klass
+ test_klass_is_empty_inline_type(elem_klass, dst_temp, empty_value);
+
+ // calc source into "array_klass" and free up some regs
+ const Register src = array_klass;
+ push(index); // preserve index reg in case alloc_failed
+ data_for_value_array_index(array, array_klass, index, src);
+
+ allocate_instance(elem_klass, obj, alloc_temp, dst_temp, false, alloc_failed);
+ // Have an oop instance buffer, copy into it
+ store_ptr(0, obj); // preserve obj (overwrite index, no longer needed)
+ data_for_oop(obj, dst_temp, elem_klass);
+ access_value_copy(IS_DEST_UNINITIALIZED, src, dst_temp, elem_klass);
+ pop(obj);
+ jmp(done);
+
+ bind(empty_value);
+ get_empty_inline_type_oop(elem_klass, dst_temp, obj);
+ jmp(done);
+
+ bind(alloc_failed);
+ pop(index);
+ if (array == c_rarg2) {
+ mov(elem_klass, array);
+ array = elem_klass;
+ }
+ call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::value_array_load), array, index);
+
+ bind(done);
+ }
+
// Lock object
//
// Args:
// rdx, c_rarg1: BasicObjectLock to be used for locking
// Load immediate 1 into swap_reg %rax
movl(swap_reg, 1);
// Load (object->mark() | 1) into swap_reg %rax
orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
+ if (EnableValhalla) {
+ // Mask inline_type bit such that we go to the slow path if object is an inline type
+ andptr(swap_reg, ~((int) markWord::inline_type_bit_in_place));
+ }
// Save (object->mark() | 1) into BasicLock's displaced header
movptr(Address(lock_reg, mark_offset), swap_reg);
assert(lock_offset == 0,
bind(profile_continue);
}
}
! void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
if (ProfileInterpreter) {
Label profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
bind(profile_continue);
}
}
! void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp, bool acmp) {
if (ProfileInterpreter) {
Label profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
// We are taking a branch. Increment the not taken count.
increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
// The method data pointer needs to be updated to correspond to
// the next bytecode
! update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));
bind(profile_continue);
}
}
void InterpreterMacroAssembler::profile_call(Register mdp) {
// We are taking a branch. Increment the not taken count.
increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()));
// The method data pointer needs to be updated to correspond to
// the next bytecode
! update_mdp_by_constant(mdp, acmp ? in_bytes(ACmpData::acmp_data_size()): in_bytes(BranchData::branch_data_size()));
bind(profile_continue);
}
}
void InterpreterMacroAssembler::profile_call(Register mdp) {
jmp(skip_receiver_profile);
bind(not_null);
}
// Record the receiver type.
! record_klass_in_profile(receiver, mdp, reg2, true);
bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
bind(profile_continue);
jmp(skip_receiver_profile);
bind(not_null);
}
// Record the receiver type.
! record_klass_in_profile(receiver, mdp, reg2);
bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
bind(profile_continue);
// Because there are two degrees of freedom in the state, a simple linear
// search will not work; it must be a decision tree. Hence this helper
// function is recursive, to generate the required tree structured code.
// It's the interpreter, so we are trading off code space for speed.
// See below for example code.
! void InterpreterMacroAssembler::record_klass_in_profile_helper(
! Register receiver, Register mdp,
! Register reg2, int start_row,
- Label& done, bool is_virtual_call) {
if (TypeProfileWidth == 0) {
increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
} else {
record_item_in_profile_helper(receiver, mdp, reg2, 0, done, TypeProfileWidth,
&VirtualCallData::receiver_offset, &VirtualCallData::receiver_count_offset);
// Because there are two degrees of freedom in the state, a simple linear
// search will not work; it must be a decision tree. Hence this helper
// function is recursive, to generate the required tree structured code.
// It's the interpreter, so we are trading off code space for speed.
// See below for example code.
! void InterpreterMacroAssembler::record_klass_in_profile_helper(Register receiver, Register mdp,
! Register reg2, int start_row,
! Label& done) {
if (TypeProfileWidth == 0) {
increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
} else {
record_item_in_profile_helper(receiver, mdp, reg2, 0, done, TypeProfileWidth,
&VirtualCallData::receiver_offset, &VirtualCallData::receiver_count_offset);
// if (row[2].rec == rec) { row[2].incr(); goto done; }
// row[0].init(rec); goto done;
// }
// done:
! void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
- Register mdp, Register reg2,
- bool is_virtual_call) {
assert(ProfileInterpreter, "must be profiling");
Label done;
! record_klass_in_profile_helper(receiver, mdp, reg2, 0, done, is_virtual_call);
bind (done);
}
void InterpreterMacroAssembler::profile_ret(Register return_bci,
// if (row[2].rec == rec) { row[2].incr(); goto done; }
// row[0].init(rec); goto done;
// }
// done:
! void InterpreterMacroAssembler::record_klass_in_profile(Register receiver, Register mdp, Register reg2) {
assert(ProfileInterpreter, "must be profiling");
Label done;
! record_klass_in_profile_helper(receiver, mdp, reg2, 0, done);
bind (done);
}
void InterpreterMacroAssembler::profile_ret(Register return_bci,
int mdp_delta = in_bytes(BitData::bit_data_size());
if (TypeProfileCasts) {
mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
// Record the object type.
! record_klass_in_profile(klass, mdp, reg2, false);
NOT_LP64(assert(reg2 == rdi, "we know how to fix this blown reg");)
NOT_LP64(restore_locals();) // Restore EDI
}
update_mdp_by_constant(mdp, mdp_delta);
int mdp_delta = in_bytes(BitData::bit_data_size());
if (TypeProfileCasts) {
mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
// Record the object type.
! record_klass_in_profile(klass, mdp, reg2);
NOT_LP64(assert(reg2 == rdi, "we know how to fix this blown reg");)
NOT_LP64(restore_locals();) // Restore EDI
}
update_mdp_by_constant(mdp, mdp_delta);
bind(profile_continue);
}
}
+ template <class ArrayData> void InterpreterMacroAssembler::profile_array_type(Register mdp,
+ Register array,
+ Register tmp) {
+ if (ProfileInterpreter) {
+ Label profile_continue;
+
+ // If no method data exists, go to profile_continue.
+ test_method_data_pointer(mdp, profile_continue);
+
+ mov(tmp, array);
+ profile_obj_type(tmp, Address(mdp, in_bytes(ArrayData::array_offset())));
+
+ Label not_flat;
+ test_non_flat_array_oop(array, tmp, not_flat);
+
+ set_mdp_flag_at(mdp, ArrayData::flat_array_byte_constant());
+
+ bind(not_flat);
+
+ Label not_null_free;
+ test_non_null_free_array_oop(array, tmp, not_null_free);
+
+ set_mdp_flag_at(mdp, ArrayData::null_free_array_byte_constant());
+
+ bind(not_null_free);
+
+ bind(profile_continue);
+ }
+ }
+
+ template void InterpreterMacroAssembler::profile_array_type<ArrayLoadData>(Register mdp,
+ Register array,
+ Register tmp);
+ template void InterpreterMacroAssembler::profile_array_type<ArrayStoreData>(Register mdp,
+ Register array,
+ Register tmp);
+
+
+ void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2) {
+ if (ProfileInterpreter) {
+ Label profile_continue;
+
+ // If no method data exists, go to profile_continue.
+ test_method_data_pointer(mdp, profile_continue);
+
+ Label done, update;
+ testptr(element, element);
+ jccb(Assembler::notZero, update);
+ set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
+ jmp(done);
+
+ bind(update);
+ load_klass(tmp, element, rscratch1);
+
+ // Record the object type.
+ record_klass_in_profile(tmp, mdp, tmp2);
+
+ bind(done);
+
+ // The method data pointer needs to be updated.
+ update_mdp_by_constant(mdp, in_bytes(ArrayStoreData::array_store_data_size()));
+
+ bind(profile_continue);
+ }
+ }
+
+ void InterpreterMacroAssembler::profile_element_type(Register mdp,
+ Register element,
+ Register tmp) {
+ if (ProfileInterpreter) {
+ Label profile_continue;
+
+ // If no method data exists, go to profile_continue.
+ test_method_data_pointer(mdp, profile_continue);
+
+ mov(tmp, element);
+ profile_obj_type(tmp, Address(mdp, in_bytes(ArrayLoadData::element_offset())));
+
+ // The method data pointer needs to be updated.
+ update_mdp_by_constant(mdp, in_bytes(ArrayLoadData::array_load_data_size()));
+
+ bind(profile_continue);
+ }
+ }
+
+ void InterpreterMacroAssembler::profile_acmp(Register mdp,
+ Register left,
+ Register right,
+ Register tmp) {
+ if (ProfileInterpreter) {
+ Label profile_continue;
+
+ // If no method data exists, go to profile_continue.
+ test_method_data_pointer(mdp, profile_continue);
+
+ mov(tmp, left);
+ profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::left_offset())));
+
+ Label left_not_inline_type;
+ test_oop_is_not_inline_type(left, tmp, left_not_inline_type);
+ set_mdp_flag_at(mdp, ACmpData::left_inline_type_byte_constant());
+ bind(left_not_inline_type);
+
+ mov(tmp, right);
+ profile_obj_type(tmp, Address(mdp, in_bytes(ACmpData::right_offset())));
+
+ Label right_not_inline_type;
+ test_oop_is_not_inline_type(right, tmp, right_not_inline_type);
+ set_mdp_flag_at(mdp, ACmpData::right_inline_type_byte_constant());
+ bind(right_not_inline_type);
+
+ bind(profile_continue);
+ }
+ }
void InterpreterMacroAssembler::_interp_verify_oop(Register reg, TosState state, const char* file, int line) {
if (state == atos) {
MacroAssembler::_verify_oop_checked(reg, "broken oop", file, line);
< prev index next >