< prev index next > src/hotspot/cpu/x86/templateTable_x86.cpp
Print this page
#include "memory/universe.hpp"
#include "oops/methodCounters.hpp"
#include "oops/methodData.hpp"
#include "oops/objArrayKlass.hpp"
#include "oops/oop.inline.hpp"
+ #include "oops/inlineKlass.hpp"
#include "oops/resolvedFieldEntry.hpp"
#include "oops/resolvedIndyEntry.hpp"
#include "oops/resolvedMethodEntry.hpp"
#include "prims/jvmtiExport.hpp"
#include "prims/methodHandles.hpp"
int byte_no) {
if (!RewriteBytecodes) return;
Label L_patch_done;
switch (bc) {
+ case Bytecodes::_fast_vputfield:
case Bytecodes::_fast_aputfield:
case Bytecodes::_fast_bputfield:
case Bytecodes::_fast_zputfield:
case Bytecodes::_fast_cputfield:
case Bytecodes::_fast_dputfield:
noreg, noreg);
}
void TemplateTable::aaload() {
transition(itos, atos);
! // rax: index
! // rdx: array
! index_check(rdx, rax); // kills rbx
! do_oop_load(_masm,
! Address(rdx, rax,
! UseCompressedOops ? Address::times_4 : Address::times_ptr,
! arrayOopDesc::base_offset_in_bytes(T_OBJECT)),
! rax,
! IS_ARRAY);
}
void TemplateTable::baload() {
transition(itos, itos);
// rax: index
noreg, noreg);
}
void TemplateTable::aaload() {
transition(itos, atos);
! Register array = rdx;
! Register index = rax;
!
! index_check(array, index); // kills rbx
! __ profile_array_type<ArrayLoadData>(rbx, array, rcx);
! if (UseFlatArray) {
! Label is_flat_array, done;
! __ test_flat_array_oop(array, rbx, is_flat_array);
! do_oop_load(_masm,
+ Address(array, index,
+ UseCompressedOops ? Address::times_4 : Address::times_ptr,
+ arrayOopDesc::base_offset_in_bytes(T_OBJECT)),
+ rax,
+ IS_ARRAY);
+ __ jmp(done);
+ __ bind(is_flat_array);
+ __ read_flat_element(array, index, rbx, rcx, rax);
+ __ bind(done);
+ } else {
+ do_oop_load(_masm,
+ Address(array, index,
+ UseCompressedOops ? Address::times_4 : Address::times_ptr,
+ arrayOopDesc::base_offset_in_bytes(T_OBJECT)),
+ rax,
+ IS_ARRAY);
+ }
+ __ profile_element_type(rbx, rax, rcx);
}
void TemplateTable::baload() {
transition(itos, itos);
// rax: index
arrayOopDesc::base_offset_in_bytes(T_DOUBLE)),
noreg /* dtos */, noreg, noreg, noreg);
}
void TemplateTable::aastore() {
! Label is_null, ok_is_subtype, done;
transition(vtos, vtos);
// stack: ..., array, index, value
__ movptr(rax, at_tos()); // value
__ movl(rcx, at_tos_p1()); // index
__ movptr(rdx, at_tos_p2()); // array
arrayOopDesc::base_offset_in_bytes(T_DOUBLE)),
noreg /* dtos */, noreg, noreg, noreg);
}
void TemplateTable::aastore() {
! Label is_null, is_flat_array, ok_is_subtype, done;
transition(vtos, vtos);
// stack: ..., array, index, value
__ movptr(rax, at_tos()); // value
__ movl(rcx, at_tos_p1()); // index
__ movptr(rdx, at_tos_p2()); // array
Address element_address(rdx, rcx,
UseCompressedOops? Address::times_4 : Address::times_ptr,
arrayOopDesc::base_offset_in_bytes(T_OBJECT));
index_check_without_pop(rdx, rcx); // kills rbx
__ testptr(rax, rax);
__ jcc(Assembler::zero, is_null);
// Move subklass into rbx
__ load_klass(rbx, rax, rscratch1);
! // Move superklass into rax
! __ load_klass(rax, rdx, rscratch1);
- __ movptr(rax, Address(rax,
ObjArrayKlass::element_klass_offset()));
// Generate subtype check. Blows rcx, rdi
// Superklass in rax. Subklass in rbx.
! __ gen_subtype_check(rbx, ok_is_subtype);
// Come here on failure
// object is at TOS
__ jump(RuntimeAddress(Interpreter::_throw_ArrayStoreException_entry));
Address element_address(rdx, rcx,
UseCompressedOops? Address::times_4 : Address::times_ptr,
arrayOopDesc::base_offset_in_bytes(T_OBJECT));
index_check_without_pop(rdx, rcx); // kills rbx
+
+ __ profile_array_type<ArrayStoreData>(rdi, rdx, rbx);
+ __ profile_multiple_element_types(rdi, rax, rbx, rcx);
+
__ testptr(rax, rax);
__ jcc(Assembler::zero, is_null);
+ // Move array class to rdi
+ __ load_klass(rdi, rdx, rscratch1);
+ if (UseFlatArray) {
+ __ movl(rbx, Address(rdi, Klass::layout_helper_offset()));
+ __ test_flat_array_layout(rbx, is_flat_array);
+ }
+
// Move subklass into rbx
__ load_klass(rbx, rax, rscratch1);
! // Move array element superklass into rax
! __ movptr(rax, Address(rdi,
ObjArrayKlass::element_klass_offset()));
// Generate subtype check. Blows rcx, rdi
// Superklass in rax. Subklass in rbx.
! // is "rbx <: rax" ? (value subclass <: array element superclass)
+ __ gen_subtype_check(rbx, ok_is_subtype, false);
// Come here on failure
// object is at TOS
__ jump(RuntimeAddress(Interpreter::_throw_ArrayStoreException_entry));
do_oop_store(_masm, element_address, rax, IS_ARRAY);
__ jmp(done);
// Have a null in rax, rdx=array, ecx=index. Store null at ary[idx]
__ bind(is_null);
! __ profile_null_seen(rbx);
// Store a null
do_oop_store(_masm, element_address, noreg, IS_ARRAY);
// Pop stack arguments
__ bind(done);
__ addptr(rsp, 3 * Interpreter::stackElementSize);
}
do_oop_store(_masm, element_address, rax, IS_ARRAY);
__ jmp(done);
// Have a null in rax, rdx=array, ecx=index. Store null at ary[idx]
__ bind(is_null);
! if (EnableValhalla) {
+ Label is_null_into_value_array_npe, store_null;
+ // No way to store null in null-free array
+ __ test_null_free_array_oop(rdx, rbx, is_null_into_value_array_npe);
+ __ jmp(store_null);
+
+ __ bind(is_null_into_value_array_npe);
+ __ jump(RuntimeAddress(Interpreter::_throw_NullPointerException_entry));
+
+ __ bind(store_null);
+ }
// Store a null
do_oop_store(_masm, element_address, noreg, IS_ARRAY);
+ __ jmp(done);
+ if (UseFlatArray) {
+ Label is_type_ok;
+ __ bind(is_flat_array); // Store non-null value to flat
+
+ // Simplistic type check...
+
+ // Profile the not-null value's klass.
+ __ load_klass(rbx, rax, rscratch1);
+ // Move element klass into rax
+ __ movptr(rax, Address(rdi, ArrayKlass::element_klass_offset()));
+ // flat value array needs exact type match
+ // is "rax == rbx" (value subclass == array element superclass)
+ __ cmpptr(rax, rbx);
+ __ jccb(Assembler::equal, is_type_ok);
+
+ __ jump(RuntimeAddress(Interpreter::_throw_ArrayStoreException_entry));
+
+ __ bind(is_type_ok);
+ // rbx: value's klass
+ // rdx: array
+ // rdi: array klass
+ __ test_klass_is_empty_inline_type(rbx, rax, done);
+
+ // calc dst for copy
+ __ movl(rax, at_tos_p1()); // index
+ __ data_for_value_array_index(rdx, rdi, rax, rax);
+
+ // ...and src for copy
+ __ movptr(rcx, at_tos()); // value
+ __ data_for_oop(rcx, rcx, rbx);
+
+ __ access_value_copy(IN_HEAP, rcx, rax, rbx);
+ }
// Pop stack arguments
__ bind(done);
__ addptr(rsp, 3 * Interpreter::stackElementSize);
}
}
void TemplateTable::if_acmp(Condition cc) {
transition(atos, vtos);
// assume branch is more often taken than not (loops use backward branches)
! Label not_taken;
__ pop_ptr(rdx);
__ cmpoop(rdx, rax);
__ jcc(j_not(cc), not_taken);
branch(false, false);
__ bind(not_taken);
! __ profile_not_taken_branch(rax);
}
void TemplateTable::ret() {
transition(vtos, vtos);
locals_index(rbx);
}
void TemplateTable::if_acmp(Condition cc) {
transition(atos, vtos);
// assume branch is more often taken than not (loops use backward branches)
! Label taken, not_taken;
__ pop_ptr(rdx);
+
+ __ profile_acmp(rbx, rdx, rax, rcx);
+
+ const int is_inline_type_mask = markWord::inline_type_pattern;
+ if (EnableValhalla) {
+ __ cmpoop(rdx, rax);
+ __ jcc(Assembler::equal, (cc == equal) ? taken : not_taken);
+
+ // might be substitutable, test if either rax or rdx is null
+ __ testptr(rax, rax);
+ __ jcc(Assembler::zero, (cc == equal) ? not_taken : taken);
+ __ testptr(rdx, rdx);
+ __ jcc(Assembler::zero, (cc == equal) ? not_taken : taken);
+
+ // and both are values ?
+ __ movptr(rbx, Address(rdx, oopDesc::mark_offset_in_bytes()));
+ __ andptr(rbx, Address(rax, oopDesc::mark_offset_in_bytes()));
+ __ andptr(rbx, is_inline_type_mask);
+ __ cmpptr(rbx, is_inline_type_mask);
+ __ jcc(Assembler::notEqual, (cc == equal) ? not_taken : taken);
+
+ // same value klass ?
+ __ load_metadata(rbx, rdx);
+ __ load_metadata(rcx, rax);
+ __ cmpptr(rbx, rcx);
+ __ jcc(Assembler::notEqual, (cc == equal) ? not_taken : taken);
+
+ // Know both are the same type, let's test for substitutability...
+ if (cc == equal) {
+ invoke_is_substitutable(rax, rdx, taken, not_taken);
+ } else {
+ invoke_is_substitutable(rax, rdx, not_taken, taken);
+ }
+ __ stop("Not reachable");
+ }
+
__ cmpoop(rdx, rax);
__ jcc(j_not(cc), not_taken);
+ __ bind(taken);
branch(false, false);
__ bind(not_taken);
! __ profile_not_taken_branch(rax, true);
+ }
+
+ void TemplateTable::invoke_is_substitutable(Register aobj, Register bobj,
+ Label& is_subst, Label& not_subst) {
+ __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::is_substitutable), aobj, bobj);
+ // Restored...rax answer, jmp to outcome...
+ __ testl(rax, rax);
+ __ jcc(Assembler::zero, not_subst);
+ __ jmp(is_subst);
}
void TemplateTable::ret() {
transition(vtos, vtos);
locals_index(rbx);
// Need to narrow in the return bytecode rather than in generate_return_entry
// since compiled code callers expect the result to already be narrowed.
if (state == itos) {
__ narrow(rax);
}
! __ remove_activation(state, rbcp);
__ jmp(rbcp);
}
// ----------------------------------------------------------------------------
// Need to narrow in the return bytecode rather than in generate_return_entry
// since compiled code callers expect the result to already be narrowed.
if (state == itos) {
__ narrow(rax);
}
!
+ __ remove_activation(state, rbcp, true, true, true);
__ jmp(rbcp);
}
// ----------------------------------------------------------------------------
}
void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
transition(vtos, vtos);
! const Register obj = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
const Register cache = rcx;
const Register index = rdx;
const Register off = rbx;
const Register tos_state = rax;
const Register flags = rdx;
}
void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
transition(vtos, vtos);
! const Register obj = LP64_ONLY(r9) NOT_LP64(rcx);
const Register cache = rcx;
const Register index = rdx;
const Register off = rbx;
const Register tos_state = rax;
const Register flags = rdx;
resolve_cache_and_index_for_field(byte_no, cache, index);
jvmti_post_field_access(cache, index, is_static, false);
load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
- if (!is_static) pop_and_check_object(obj);
-
const Address field(obj, off, Address::times_1, 0*wordSize);
! Label Done, notByte, notBool, notInt, notShort, notChar, notLong, notFloat, notObj;
// Make sure we don't need to mask edx after the above shift
assert(btos == 0, "change code, btos != 0");
__ testl(tos_state, tos_state);
__ jcc(Assembler::notZero, notByte);
// btos
__ access_load_at(T_BYTE, IN_HEAP, rax, field, noreg, noreg);
__ push(btos);
// Rewrite bytecode to be faster
if (!is_static && rc == may_rewrite) {
patch_bytecode(Bytecodes::_fast_bgetfield, bc, rbx);
resolve_cache_and_index_for_field(byte_no, cache, index);
jvmti_post_field_access(cache, index, is_static, false);
load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
const Address field(obj, off, Address::times_1, 0*wordSize);
! Label Done, notByte, notBool, notInt, notShort, notChar, notLong, notFloat, notObj, notInlineType;
// Make sure we don't need to mask edx after the above shift
assert(btos == 0, "change code, btos != 0");
__ testl(tos_state, tos_state);
__ jcc(Assembler::notZero, notByte);
// btos
+ if (!is_static) pop_and_check_object(obj);
__ access_load_at(T_BYTE, IN_HEAP, rax, field, noreg, noreg);
__ push(btos);
// Rewrite bytecode to be faster
if (!is_static && rc == may_rewrite) {
patch_bytecode(Bytecodes::_fast_bgetfield, bc, rbx);
__ jmp(Done);
__ bind(notByte);
__ cmpl(tos_state, ztos);
__ jcc(Assembler::notEqual, notBool);
!
// ztos (same code as btos)
__ access_load_at(T_BOOLEAN, IN_HEAP, rax, field, noreg, noreg);
__ push(ztos);
// Rewrite bytecode to be faster
if (!is_static && rc == may_rewrite) {
__ jmp(Done);
__ bind(notByte);
__ cmpl(tos_state, ztos);
__ jcc(Assembler::notEqual, notBool);
! if (!is_static) pop_and_check_object(obj);
// ztos (same code as btos)
__ access_load_at(T_BOOLEAN, IN_HEAP, rax, field, noreg, noreg);
__ push(ztos);
// Rewrite bytecode to be faster
if (!is_static && rc == may_rewrite) {
__ bind(notBool);
__ cmpl(tos_state, atos);
__ jcc(Assembler::notEqual, notObj);
// atos
! do_oop_load(_masm, field, rax);
! __ push(atos);
! if (!is_static && rc == may_rewrite) {
! patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx);
}
- __ jmp(Done);
__ bind(notObj);
__ cmpl(tos_state, itos);
__ jcc(Assembler::notEqual, notInt);
// itos
__ access_load_at(T_INT, IN_HEAP, rax, field, noreg, noreg);
__ push(itos);
__ bind(notBool);
__ cmpl(tos_state, atos);
__ jcc(Assembler::notEqual, notObj);
// atos
! if (!EnableValhalla) {
! if (!is_static) pop_and_check_object(obj);
! do_oop_load(_masm, field, rax);
! __ push(atos);
+ if (!is_static && rc == may_rewrite) {
+ patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx);
+ }
+ __ jmp(Done);
+ } else {
+ if (is_static) {
+ __ load_heap_oop(rax, field);
+ Label is_null_free_inline_type, uninitialized;
+ // Issue below if the static field has not been initialized yet
+ __ test_field_is_null_free_inline_type(flags, rscratch1, is_null_free_inline_type);
+ // field is not a null free inline type
+ __ push(atos);
+ __ jmp(Done);
+ // field is a null free inline type, must not return null even if uninitialized
+ __ bind(is_null_free_inline_type);
+ __ testptr(rax, rax);
+ __ jcc(Assembler::zero, uninitialized);
+ __ push(atos);
+ __ jmp(Done);
+ __ bind(uninitialized);
+ #ifdef _LP64
+ Label slow_case, finish;
+ __ movptr(rbx, Address(obj, java_lang_Class::klass_offset()));
+ __ cmpb(Address(rbx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
+ __ jcc(Assembler::notEqual, slow_case);
+ __ get_default_value_oop(rbx, rscratch1, rax);
+ __ jmp(finish);
+ __ bind(slow_case);
+ #endif // LP64
+ __ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_static_inline_type_field),
+ obj, cache);
+ #ifdef _LP64
+ __ bind(finish);
+ #endif // _LP64
+ __ verify_oop(rax);
+ __ push(atos);
+ __ jmp(Done);
+ } else {
+ Label is_flat, nonnull, is_inline_type, rewrite_inline, has_null_marker;
+ __ test_field_is_null_free_inline_type(flags, rscratch1, is_inline_type);
+ __ test_field_has_null_marker(flags, rscratch1, has_null_marker);
+ // field is not a null free inline type
+ pop_and_check_object(obj);
+ __ load_heap_oop(rax, field);
+ __ push(atos);
+ if (rc == may_rewrite) {
+ patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx);
+ }
+ __ jmp(Done);
+ __ bind(is_inline_type);
+ __ test_field_is_flat(flags, rscratch1, is_flat);
+ // field is not flat
+ pop_and_check_object(obj);
+ __ load_heap_oop(rax, field);
+ __ testptr(rax, rax);
+ __ jcc(Assembler::notZero, nonnull);
+ __ load_unsigned_short(flags, Address(cache, in_bytes(ResolvedFieldEntry::field_index_offset())));
+ __ movptr(rcx, Address(cache, ResolvedFieldEntry::field_holder_offset()));
+ __ get_inline_type_field_klass(rcx, flags, rbx);
+ __ get_default_value_oop(rbx, rcx, rax);
+ __ bind(nonnull);
+ __ verify_oop(rax);
+ __ push(atos);
+ __ jmp(rewrite_inline);
+ __ bind(is_flat);
+ pop_and_check_object(rax);
+ __ read_flat_field(rcx, rdx, rbx, rax);
+ __ verify_oop(rax);
+ __ push(atos);
+ __ jmp(rewrite_inline);
+ __ bind(has_null_marker);
+ pop_and_check_object(rax);
+ __ load_field_entry(rcx, rbx);
+ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_nullable_flat_field), rax, rcx);
+ __ get_vm_result(rax, r15_thread);
+ __ push(atos);
+ __ bind(rewrite_inline);
+ if (rc == may_rewrite) {
+ patch_bytecode(Bytecodes::_fast_vgetfield, bc, rbx);
+ }
+ __ jmp(Done);
+ }
}
__ bind(notObj);
+
+ if (!is_static) pop_and_check_object(obj);
+
__ cmpl(tos_state, itos);
__ jcc(Assembler::notEqual, notInt);
// itos
__ access_load_at(T_INT, IN_HEAP, rax, field, noreg, noreg);
__ push(itos);
void TemplateTable::getstatic(int byte_no) {
getfield_or_static(byte_no, true);
}
-
// The registers cache and index expected to be set before call.
// The function may destroy various registers, just not the cache and index registers.
void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
// Cache is rcx and index is rdx
const Register entry = LP64_ONLY(c_rarg2) NOT_LP64(rax); // ResolvedFieldEntry
const Register obj = rcx;
const Register cache = rcx;
const Register index = rdx;
const Register tos_state = rdx;
const Register off = rbx;
! const Register flags = rax;
resolve_cache_and_index_for_field(byte_no, cache, index);
jvmti_post_field_mod(cache, index, is_static);
load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
const Register obj = rcx;
const Register cache = rcx;
const Register index = rdx;
const Register tos_state = rdx;
const Register off = rbx;
! const Register flags = r9;
resolve_cache_and_index_for_field(byte_no, cache, index);
jvmti_post_field_mod(cache, index, is_static);
load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
// Assembler::StoreStore));
Label notVolatile, Done;
// Check for volatile store
! __ andl(flags, (1 << ResolvedFieldEntry::is_volatile_shift));
! __ testl(flags, flags);
__ jcc(Assembler::zero, notVolatile);
! putfield_or_static_helper(byte_no, is_static, rc, obj, off, tos_state);
volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
Assembler::StoreStore));
__ jmp(Done);
__ bind(notVolatile);
! putfield_or_static_helper(byte_no, is_static, rc, obj, off, tos_state);
__ bind(Done);
}
void TemplateTable::putfield_or_static_helper(int byte_no, bool is_static, RewriteControl rc,
! Register obj, Register off, Register tos_state) {
// field addresses
const Address field(obj, off, Address::times_1, 0*wordSize);
NOT_LP64( const Address hi(obj, off, Address::times_1, 1*wordSize);)
Label notByte, notBool, notInt, notShort, notChar,
! notLong, notFloat, notObj;
Label Done;
const Register bc = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
// Test TOS state
// Assembler::StoreStore));
Label notVolatile, Done;
// Check for volatile store
! __ movl(rscratch1, flags);
! __ andl(rscratch1, (1 << ResolvedFieldEntry::is_volatile_shift));
+ __ testl(rscratch1, rscratch1);
__ jcc(Assembler::zero, notVolatile);
! putfield_or_static_helper(byte_no, is_static, rc, obj, off, tos_state, flags);
volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
Assembler::StoreStore));
__ jmp(Done);
__ bind(notVolatile);
! putfield_or_static_helper(byte_no, is_static, rc, obj, off, tos_state, flags);
__ bind(Done);
}
void TemplateTable::putfield_or_static_helper(int byte_no, bool is_static, RewriteControl rc,
! Register obj, Register off, Register tos_state, Register flags) {
// field addresses
const Address field(obj, off, Address::times_1, 0*wordSize);
NOT_LP64( const Address hi(obj, off, Address::times_1, 1*wordSize);)
Label notByte, notBool, notInt, notShort, notChar,
! notLong, notFloat, notObj, notInlineType;
Label Done;
const Register bc = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
// Test TOS state
__ cmpl(tos_state, atos);
__ jcc(Assembler::notEqual, notObj);
// atos
{
! __ pop(atos);
! if (!is_static) pop_and_check_object(obj);
! // Store into the field
! do_oop_store(_masm, field, rax);
! if (!is_static && rc == may_rewrite) {
! patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no);
}
- __ jmp(Done);
}
__ bind(notObj);
__ cmpl(tos_state, itos);
__ jcc(Assembler::notEqual, notInt);
__ cmpl(tos_state, atos);
__ jcc(Assembler::notEqual, notObj);
// atos
{
! if (!EnableValhalla) {
! __ pop(atos);
! if (!is_static) pop_and_check_object(obj);
! // Store into the field
! do_oop_store(_masm, field, rax);
! if (!is_static && rc == may_rewrite) {
+ patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no);
+ }
+ __ jmp(Done);
+ } else {
+ __ pop(atos);
+ if (is_static) {
+ Label is_inline_type;
+ __ test_field_is_not_null_free_inline_type(flags, rscratch1, is_inline_type);
+ __ null_check(rax);
+ __ bind(is_inline_type);
+ do_oop_store(_masm, field, rax);
+ __ jmp(Done);
+ } else {
+ Label is_null_free_inline_type, is_flat, has_null_marker,
+ write_null, rewrite_not_inline, rewrite_inline;
+ __ test_field_is_null_free_inline_type(flags, rscratch1, is_null_free_inline_type);
+ __ test_field_has_null_marker(flags, rscratch1, has_null_marker);
+ // Not an inline type
+ pop_and_check_object(obj);
+ // Store into the field
+ do_oop_store(_masm, field, rax);
+ __ bind(rewrite_not_inline);
+ if (rc == may_rewrite) {
+ patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no);
+ }
+ __ jmp(Done);
+ // Implementation of the inline type semantic
+ __ bind(is_null_free_inline_type);
+ __ null_check(rax);
+ __ test_field_is_flat(flags, rscratch1, is_flat);
+ // field is not flat
+ pop_and_check_object(obj);
+ // Store into the field
+ do_oop_store(_masm, field, rax);
+ __ jmp(rewrite_inline);
+ __ bind(is_flat);
+ // field is flat
+ __ load_unsigned_short(rdx, Address(rcx, in_bytes(ResolvedFieldEntry::field_index_offset())));
+ __ movptr(r9, Address(rcx, in_bytes(ResolvedFieldEntry::field_holder_offset())));
+ pop_and_check_object(obj); // obj = rcx
+ __ load_klass(r8, rax, rscratch1);
+ __ data_for_oop(rax, rax, r8);
+ __ addptr(obj, off);
+ __ inline_layout_info(r9, rdx, rbx);
+ // because we use InlineLayoutInfo, we need special value access code specialized for fields (arrays will need a different API)
+ __ flat_field_copy(IN_HEAP, rax, obj, rbx);
+ __ jmp(rewrite_inline);
+ __ bind(has_null_marker); // has null marker means the field is flat with a null marker
+ pop_and_check_object(rbx);
+ __ load_field_entry(rcx, rdx);
+ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_nullable_flat_field), rbx, rax, rcx);
+ __ bind(rewrite_inline);
+ if (rc == may_rewrite) {
+ patch_bytecode(Bytecodes::_fast_vputfield, bc, rbx, true, byte_no);
+ }
+ __ jmp(Done);
+ }
}
}
__ bind(notObj);
__ cmpl(tos_state, itos);
__ jcc(Assembler::notEqual, notInt);
__ push_ptr(rbx); // put the object pointer back on tos
// Save tos values before call_VM() clobbers them. Since we have
// to do it for every data type, we use the saved values as the
// jvalue object.
switch (bytecode()) { // load values into the jvalue object
+ case Bytecodes::_fast_vputfield: //fall through
case Bytecodes::_fast_aputfield: __ push_ptr(rax); break;
case Bytecodes::_fast_bputfield: // fall through
case Bytecodes::_fast_zputfield: // fall through
case Bytecodes::_fast_sputfield: // fall through
case Bytecodes::_fast_cputfield: // fall through
// c_rarg3: jvalue object on the stack
LP64_ONLY(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification), rbx, c_rarg2, c_rarg3));
NOT_LP64(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification), rbx, rax, rcx));
switch (bytecode()) { // restore tos values
+ case Bytecodes::_fast_vputfield: // fall through
case Bytecodes::_fast_aputfield: __ pop_ptr(rax); break;
case Bytecodes::_fast_bputfield: // fall through
case Bytecodes::_fast_zputfield: // fall through
case Bytecodes::_fast_sputfield: // fall through
case Bytecodes::_fast_cputfield: // fall through
}
void TemplateTable::fast_storefield(TosState state) {
transition(state, vtos);
- Register cache = rcx;
-
Label notVolatile, Done;
jvmti_post_fast_field_mod();
__ push(rax);
__ load_field_entry(rcx, rax);
! load_resolved_field_entry(noreg, cache, rax, rbx, rdx);
- // RBX: field offset, RAX: TOS, RDX: flags
- __ andl(rdx, (1 << ResolvedFieldEntry::is_volatile_shift));
__ pop(rax);
// Get object from stack
pop_and_check_object(rcx);
// field address
const Address field(rcx, rbx, Address::times_1);
// Check for volatile store
! __ testl(rdx, rdx);
__ jcc(Assembler::zero, notVolatile);
! fast_storefield_helper(field, rax);
volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
Assembler::StoreStore));
__ jmp(Done);
__ bind(notVolatile);
! fast_storefield_helper(field, rax);
__ bind(Done);
}
! void TemplateTable::fast_storefield_helper(Address field, Register rax) {
// access field
switch (bytecode()) {
case Bytecodes::_fast_aputfield:
! do_oop_store(_masm, field, rax);
break;
case Bytecodes::_fast_lputfield:
#ifdef _LP64
__ access_store_at(T_LONG, IN_HEAP, field, noreg /* ltos */, noreg, noreg, noreg);
#else
}
void TemplateTable::fast_storefield(TosState state) {
transition(state, vtos);
Label notVolatile, Done;
jvmti_post_fast_field_mod();
__ push(rax);
__ load_field_entry(rcx, rax);
! load_resolved_field_entry(noreg, rcx, rax, rbx, rdx);
__ pop(rax);
+ // RBX: field offset, RCX: RAX: TOS, RDX: flags
// Get object from stack
pop_and_check_object(rcx);
// field address
const Address field(rcx, rbx, Address::times_1);
// Check for volatile store
! __ movl(rscratch2, rdx); // saving flags for is_flat test
+ __ andl(rscratch2, (1 << ResolvedFieldEntry::is_volatile_shift));
+ __ testl(rscratch2, rscratch2);
__ jcc(Assembler::zero, notVolatile);
! fast_storefield_helper(field, rax, rdx);
volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
Assembler::StoreStore));
__ jmp(Done);
__ bind(notVolatile);
! fast_storefield_helper(field, rax, rdx);
__ bind(Done);
}
! void TemplateTable::fast_storefield_helper(Address field, Register rax, Register flags) {
+
+ // DANGER: 'field' argument depends on rcx and rbx
// access field
switch (bytecode()) {
+ case Bytecodes::_fast_vputfield:
+ {
+ Label is_flat, has_null_marker, write_null, done;
+ __ test_field_has_null_marker(flags, rscratch1, has_null_marker);
+ // Null free field cases: flat or not flat
+ __ null_check(rax);
+ __ test_field_is_flat(flags, rscratch1, is_flat);
+ // field is not flat
+ do_oop_store(_masm, field, rax);
+ __ jmp(done);
+ __ bind(is_flat);
+ __ load_field_entry(r8, r9);
+ __ load_unsigned_short(r9, Address(r8, in_bytes(ResolvedFieldEntry::field_index_offset())));
+ __ movptr(r8, Address(r8, in_bytes(ResolvedFieldEntry::field_holder_offset())));
+ __ inline_layout_info(r8, r9, r8);
+ __ load_klass(rdx, rax, rscratch1);
+ __ data_for_oop(rax, rax, rdx);
+ __ lea(rcx, field);
+ __ flat_field_copy(IN_HEAP, rax, rcx, r8);
+ __ jmp(done);
+ __ bind(has_null_marker); // has null marker means the field is flat with a null marker
+ __ movptr(rbx, rcx);
+ __ load_field_entry(rcx, rdx);
+ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::write_nullable_flat_field), rbx, rax, rcx);
+ __ bind(done);
+ }
+ break;
case Bytecodes::_fast_aputfield:
! {
+ do_oop_store(_masm, field, rax);
+ }
break;
case Bytecodes::_fast_lputfield:
#ifdef _LP64
__ access_store_at(T_LONG, IN_HEAP, field, noreg /* ltos */, noreg, noreg, noreg);
#else
__ bind(L1);
}
// access constant pool cache
__ load_field_entry(rcx, rbx);
! __ load_sized_value(rbx, Address(rcx, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
// rax: object
__ verify_oop(rax);
__ null_check(rax);
! Address field(rax, rbx, Address::times_1);
// access field
switch (bytecode()) {
case Bytecodes::_fast_agetfield:
do_oop_load(_masm, field, rax);
__ verify_oop(rax);
break;
case Bytecodes::_fast_lgetfield:
__ bind(L1);
}
// access constant pool cache
__ load_field_entry(rcx, rbx);
! __ load_sized_value(rdx, Address(rcx, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
// rax: object
__ verify_oop(rax);
__ null_check(rax);
! Address field(rax, rdx, Address::times_1);
// access field
switch (bytecode()) {
+ case Bytecodes::_fast_vgetfield:
+ {
+ Label is_flat, nonnull, Done, has_null_marker;
+ __ load_unsigned_byte(rscratch1, Address(rcx, in_bytes(ResolvedFieldEntry::flags_offset())));
+ __ test_field_has_null_marker(rscratch1, rscratch2, has_null_marker);
+ __ test_field_is_flat(rscratch1, rscratch2, is_flat);
+ // field is not flat
+ __ load_heap_oop(rax, field);
+ __ testptr(rax, rax);
+ __ jcc(Assembler::notZero, nonnull);
+ __ load_unsigned_short(rdx, Address(rcx, in_bytes(ResolvedFieldEntry::field_index_offset())));
+ __ movptr(rcx, Address(rcx, ResolvedFieldEntry::field_holder_offset()));
+ __ get_inline_type_field_klass(rcx, rdx, rbx);
+ __ get_default_value_oop(rbx, rcx, rax);
+ __ bind(nonnull);
+ __ verify_oop(rax);
+ __ jmp(Done);
+ __ bind(is_flat);
+ // field is flat
+ __ read_flat_field(rcx, rdx, rbx, rax);
+ __ jmp(Done);
+ __ bind(has_null_marker);
+ // rax = instance, rcx = resolved entry
+ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::read_nullable_flat_field), rax, rcx);
+ __ get_vm_result(rax, r15_thread);
+ __ bind(Done);
+ __ verify_oop(rax);
+ }
+ break;
case Bytecodes::_fast_agetfield:
do_oop_load(_masm, field, rax);
__ verify_oop(rax);
break;
case Bytecodes::_fast_lgetfield:
void TemplateTable::_new() {
transition(vtos, atos);
__ get_unsigned_2_byte_index_at_bcp(rdx, 1);
Label slow_case;
- Label slow_case_no_pop;
Label done;
- Label initialize_header;
__ get_cpool_and_tags(rcx, rax);
// Make sure the class we're about to instantiate has been resolved.
// This is done before loading InstanceKlass to be consistent with the order
// how Constant Pool is updated (see ConstantPool::klass_at_put)
const int tags_offset = Array<u1>::base_offset_in_bytes();
__ cmpb(Address(rax, rdx, Address::times_1, tags_offset), JVM_CONSTANT_Class);
! __ jcc(Assembler::notEqual, slow_case_no_pop);
// get InstanceKlass
__ load_resolved_klass_at_index(rcx, rcx, rdx);
- __ push(rcx); // save the contexts of klass for initializing the header
// make sure klass is initialized
// init_state needs acquire, but x86 is TSO, and so we are already good.
#ifdef _LP64
assert(VM_Version::supports_fast_class_init_checks(), "must support fast class initialization checks");
void TemplateTable::_new() {
transition(vtos, atos);
__ get_unsigned_2_byte_index_at_bcp(rdx, 1);
Label slow_case;
Label done;
__ get_cpool_and_tags(rcx, rax);
// Make sure the class we're about to instantiate has been resolved.
// This is done before loading InstanceKlass to be consistent with the order
// how Constant Pool is updated (see ConstantPool::klass_at_put)
const int tags_offset = Array<u1>::base_offset_in_bytes();
__ cmpb(Address(rax, rdx, Address::times_1, tags_offset), JVM_CONSTANT_Class);
! __ jcc(Assembler::notEqual, slow_case);
// get InstanceKlass
__ load_resolved_klass_at_index(rcx, rcx, rdx);
// make sure klass is initialized
// init_state needs acquire, but x86 is TSO, and so we are already good.
#ifdef _LP64
assert(VM_Version::supports_fast_class_init_checks(), "must support fast class initialization checks");
#else
__ cmpb(Address(rcx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
__ jcc(Assembler::notEqual, slow_case);
#endif
! // get instance_size in InstanceKlass (scaled to a count of bytes)
- __ movl(rdx, Address(rcx, Klass::layout_helper_offset()));
- // test to see if it is malformed in some way
- __ testl(rdx, Klass::_lh_instance_slow_path_bit);
- __ jcc(Assembler::notZero, slow_case);
-
- // Allocate the instance:
- // If TLAB is enabled:
- // Try to allocate in the TLAB.
- // If fails, go to the slow path.
- // Initialize the allocation.
- // Exit.
- //
- // Go to slow path.
-
- const Register thread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
-
- if (UseTLAB) {
- NOT_LP64(__ get_thread(thread);)
- __ tlab_allocate(thread, rax, rdx, 0, rcx, rbx, slow_case);
- if (ZeroTLAB) {
- // the fields have been already cleared
- __ jmp(initialize_header);
- }
-
- // The object is initialized before the header. If the object size is
- // zero, go directly to the header initialization.
- __ decrement(rdx, sizeof(oopDesc));
- __ jcc(Assembler::zero, initialize_header);
-
- // Initialize topmost object field, divide rdx by 8, check if odd and
- // test if zero.
- __ xorl(rcx, rcx); // use zero reg to clear memory (shorter code)
- __ shrl(rdx, LogBytesPerLong); // divide by 2*oopSize and set carry flag if odd
-
- // rdx must have been multiple of 8
- #ifdef ASSERT
- // make sure rdx was multiple of 8
- Label L;
- // Ignore partial flag stall after shrl() since it is debug VM
- __ jcc(Assembler::carryClear, L);
- __ stop("object size is not multiple of 2 - adjust this code");
- __ bind(L);
- // rdx must be > 0, no extra check needed here
- #endif
-
- // initialize remaining object fields: rdx was a multiple of 8
- { Label loop;
- __ bind(loop);
- __ movptr(Address(rax, rdx, Address::times_8, sizeof(oopDesc) - 1*oopSize), rcx);
- NOT_LP64(__ movptr(Address(rax, rdx, Address::times_8, sizeof(oopDesc) - 2*oopSize), rcx));
- __ decrement(rdx);
- __ jcc(Assembler::notZero, loop);
- }
-
- // initialize object header only.
- __ bind(initialize_header);
- __ movptr(Address(rax, oopDesc::mark_offset_in_bytes()),
- (intptr_t)markWord::prototype().value()); // header
- __ pop(rcx); // get saved klass back in the register.
- #ifdef _LP64
- __ xorl(rsi, rsi); // use zero reg to clear memory (shorter code)
- __ store_klass_gap(rax, rsi); // zero klass gap for compressed oops
- #endif
- __ store_klass(rax, rcx, rscratch1); // klass
-
if (DTraceAllocProbes) {
// Trigger dtrace event for fastpath
__ push(atos);
__ call_VM_leaf(
CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), rax);
__ pop(atos);
}
!
- __ jmp(done);
- }
// slow case
__ bind(slow_case);
- __ pop(rcx); // restore stack pointer to what it was when we came in.
- __ bind(slow_case_no_pop);
Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rax);
Register rarg2 = LP64_ONLY(c_rarg2) NOT_LP64(rdx);
__ get_constant_pool(rarg1);
#else
__ cmpb(Address(rcx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
__ jcc(Assembler::notEqual, slow_case);
#endif
! __ allocate_instance(rcx, rax, rdx, rbx, true, slow_case);
if (DTraceAllocProbes) {
// Trigger dtrace event for fastpath
__ push(atos);
__ call_VM_leaf(
CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), rax);
__ pop(atos);
}
! __ jmp(done);
// slow case
__ bind(slow_case);
Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rax);
Register rarg2 = LP64_ONLY(c_rarg2) NOT_LP64(rdx);
__ get_constant_pool(rarg1);
// Get cpool & tags index
__ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
__ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
// See if bytecode has already been quicked
! __ cmpb(Address(rdx, rbx,
! Address::times_1,
! Array<u1>::base_offset_in_bytes()),
! JVM_CONSTANT_Class);
__ jcc(Assembler::equal, quicked);
__ push(atos); // save receiver for result, and for GC
call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
// vm_result_2 has metadata result
// Get cpool & tags index
__ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
__ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
// See if bytecode has already been quicked
! __ movzbl(rdx, Address(rdx, rbx,
! Address::times_1,
! Array<u1>::base_offset_in_bytes()));
! __ cmpl(rdx, JVM_CONSTANT_Class);
__ jcc(Assembler::equal, quicked);
__ push(atos); // save receiver for result, and for GC
call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
// vm_result_2 has metadata result
__ jump(RuntimeAddress(Interpreter::_throw_ClassCastException_entry));
// Come here on success
__ bind(ok_is_subtype);
__ mov(rax, rdx); // Restore object in rdx
// Collect counts on whether this check-cast sees nulls a lot or not.
if (ProfileInterpreter) {
- __ jmp(done);
- __ bind(is_null);
__ profile_null_seen(rcx);
- } else {
- __ bind(is_null); // same as 'done'
}
__ bind(done);
}
void TemplateTable::instanceof() {
transition(atos, itos);
__ jump(RuntimeAddress(Interpreter::_throw_ClassCastException_entry));
// Come here on success
__ bind(ok_is_subtype);
__ mov(rax, rdx); // Restore object in rdx
+ __ jmp(done);
+
+ __ bind(is_null);
// Collect counts on whether this check-cast sees nulls a lot or not.
if (ProfileInterpreter) {
__ profile_null_seen(rcx);
}
+
__ bind(done);
}
void TemplateTable::instanceof() {
transition(atos, itos);
// Get cpool & tags index
__ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
__ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
// See if bytecode has already been quicked
! __ cmpb(Address(rdx, rbx,
! Address::times_1,
! Array<u1>::base_offset_in_bytes()),
! JVM_CONSTANT_Class);
__ jcc(Assembler::equal, quicked);
__ push(atos); // save receiver for result, and for GC
call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
// vm_result_2 has metadata result
// Get cpool & tags index
__ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
__ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
// See if bytecode has already been quicked
! __ movzbl(rdx, Address(rdx, rbx,
! Address::times_1,
! Array<u1>::base_offset_in_bytes()));
! __ cmpl(rdx, JVM_CONSTANT_Class);
__ jcc(Assembler::equal, quicked);
__ push(atos); // save receiver for result, and for GC
call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
// vm_result_2 has metadata result
__ bind(done);
// rax = 0: obj == nullptr or obj is not an instanceof the specified klass
// rax = 1: obj != nullptr and obj is an instanceof the specified klass
}
-
//----------------------------------------------------------------------------------------------------
// Breakpoints
void TemplateTable::_breakpoint() {
// Note: We get here even if we are single stepping..
// jbug insists on setting breakpoints at every bytecode
transition(atos, vtos);
// check for null object
__ null_check(rax);
+ Label is_inline_type;
+ __ movptr(rbx, Address(rax, oopDesc::mark_offset_in_bytes()));
+ __ test_markword_is_inline_type(rbx, is_inline_type);
+
const Address monitor_block_top(
rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
const Address monitor_block_bot(
rbp, frame::interpreter_frame_initial_sp_offset * wordSize);
const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
__ generate_stack_overflow_check(0);
// The bcp has already been incremented. Just need to dispatch to
// next instruction.
__ dispatch_next(vtos);
+
+ __ bind(is_inline_type);
+ __ call_VM(noreg, CAST_FROM_FN_PTR(address,
+ InterpreterRuntime::throw_identity_exception), rax);
+ __ should_not_reach_here();
}
void TemplateTable::monitorexit() {
transition(atos, vtos);
// check for null object
__ null_check(rax);
+ const int is_inline_type_mask = markWord::inline_type_pattern;
+ Label has_identity;
+ __ movptr(rbx, Address(rax, oopDesc::mark_offset_in_bytes()));
+ __ andptr(rbx, is_inline_type_mask);
+ __ cmpl(rbx, is_inline_type_mask);
+ __ jcc(Assembler::notEqual, has_identity);
+ __ call_VM(noreg, CAST_FROM_FN_PTR(address,
+ InterpreterRuntime::throw_illegal_monitor_state_exception));
+ __ should_not_reach_here();
+ __ bind(has_identity);
+
const Address monitor_block_top(
rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
const Address monitor_block_bot(
rbp, frame::interpreter_frame_initial_sp_offset * wordSize);
const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
< prev index next >