< prev index next >

src/hotspot/cpu/x86/templateTable_x86.cpp

Print this page
*** 35,10 ***
--- 35,11 ---
  #include "memory/universe.hpp"
  #include "oops/methodCounters.hpp"
  #include "oops/methodData.hpp"
  #include "oops/objArrayKlass.hpp"
  #include "oops/oop.inline.hpp"
+ #include "oops/inlineKlass.hpp"
  #include "oops/resolvedFieldEntry.hpp"
  #include "oops/resolvedIndyEntry.hpp"
  #include "oops/resolvedMethodEntry.hpp"
  #include "prims/jvmtiExport.hpp"
  #include "prims/methodHandles.hpp"

*** 181,10 ***
--- 182,11 ---
                                     int byte_no) {
    if (!RewriteBytecodes)  return;
    Label L_patch_done;
  
    switch (bc) {
+   case Bytecodes::_fast_qputfield:
    case Bytecodes::_fast_aputfield:
    case Bytecodes::_fast_bputfield:
    case Bytecodes::_fast_zputfield:
    case Bytecodes::_fast_cputfield:
    case Bytecodes::_fast_dputfield:

*** 828,19 ***
                      noreg, noreg);
  }
  
  void TemplateTable::aaload() {
    transition(itos, atos);
!   // rax: index
!   // rdx: array
!   index_check(rdx, rax); // kills rbx
!   do_oop_load(_masm,
!               Address(rdx, rax,
!                       UseCompressedOops ? Address::times_4 : Address::times_ptr,
!                       arrayOopDesc::base_offset_in_bytes(T_OBJECT)),
!               rax,
!               IS_ARRAY);
  }
  
  void TemplateTable::baload() {
    transition(itos, itos);
    // rax: index
--- 830,37 ---
                      noreg, noreg);
  }
  
  void TemplateTable::aaload() {
    transition(itos, atos);
!   Register array = rdx;
!   Register index = rax;
! 
!   index_check(array, index); // kills rbx
!   __ profile_array_type<ArrayLoadData>(rbx, array, rcx);
!   if (UseFlatArray) {
!     Label is_flat_array, done;
!     __ test_flat_array_oop(array, rbx, is_flat_array);
!     do_oop_load(_masm,
+                 Address(array, index,
+                         UseCompressedOops ? Address::times_4 : Address::times_ptr,
+                         arrayOopDesc::base_offset_in_bytes(T_OBJECT)),
+                 rax,
+                 IS_ARRAY);
+     __ jmp(done);
+     __ bind(is_flat_array);
+     __ read_flat_element(array, index, rbx, rcx, rax);
+     __ bind(done);
+   } else {
+     do_oop_load(_masm,
+                 Address(array, index,
+                         UseCompressedOops ? Address::times_4 : Address::times_ptr,
+                         arrayOopDesc::base_offset_in_bytes(T_OBJECT)),
+                 rax,
+                 IS_ARRAY);
+   }
+   __ profile_element_type(rbx, rax, rcx);
  }
  
  void TemplateTable::baload() {
    transition(itos, itos);
    // rax: index

*** 1122,11 ***
                               arrayOopDesc::base_offset_in_bytes(T_DOUBLE)),
                       noreg /* dtos */, noreg, noreg, noreg);
  }
  
  void TemplateTable::aastore() {
!   Label is_null, ok_is_subtype, done;
    transition(vtos, vtos);
    // stack: ..., array, index, value
    __ movptr(rax, at_tos());    // value
    __ movl(rcx, at_tos_p1()); // index
    __ movptr(rdx, at_tos_p2()); // array
--- 1142,11 ---
                               arrayOopDesc::base_offset_in_bytes(T_DOUBLE)),
                       noreg /* dtos */, noreg, noreg, noreg);
  }
  
  void TemplateTable::aastore() {
!   Label is_null, is_flat_array, ok_is_subtype, done;
    transition(vtos, vtos);
    // stack: ..., array, index, value
    __ movptr(rax, at_tos());    // value
    __ movl(rcx, at_tos_p1()); // index
    __ movptr(rdx, at_tos_p2()); // array

*** 1134,23 ***
    Address element_address(rdx, rcx,
                            UseCompressedOops? Address::times_4 : Address::times_ptr,
                            arrayOopDesc::base_offset_in_bytes(T_OBJECT));
  
    index_check_without_pop(rdx, rcx);     // kills rbx
    __ testptr(rax, rax);
    __ jcc(Assembler::zero, is_null);
  
    // Move subklass into rbx
    __ load_klass(rbx, rax, rscratch1);
!   // Move superklass into rax
!   __ load_klass(rax, rdx, rscratch1);
-   __ movptr(rax, Address(rax,
                           ObjArrayKlass::element_klass_offset()));
  
    // Generate subtype check.  Blows rcx, rdi
    // Superklass in rax.  Subklass in rbx.
!   __ gen_subtype_check(rbx, ok_is_subtype);
  
    // Come here on failure
    // object is at TOS
    __ jump(ExternalAddress(Interpreter::_throw_ArrayStoreException_entry));
  
--- 1154,34 ---
    Address element_address(rdx, rcx,
                            UseCompressedOops? Address::times_4 : Address::times_ptr,
                            arrayOopDesc::base_offset_in_bytes(T_OBJECT));
  
    index_check_without_pop(rdx, rcx);     // kills rbx
+ 
+   __ profile_array_type<ArrayStoreData>(rdi, rdx, rbx);
+   __ profile_multiple_element_types(rdi, rax, rbx, rcx);
+ 
    __ testptr(rax, rax);
    __ jcc(Assembler::zero, is_null);
  
+   // Move array class to rdi
+   __ load_klass(rdi, rdx, rscratch1);
+   if (UseFlatArray) {
+     __ movl(rbx, Address(rdi, Klass::layout_helper_offset()));
+     __ test_flat_array_layout(rbx, is_flat_array);
+   }
+ 
    // Move subklass into rbx
    __ load_klass(rbx, rax, rscratch1);
!   // Move array element superklass into rax
!   __ movptr(rax, Address(rdi,
                           ObjArrayKlass::element_klass_offset()));
  
    // Generate subtype check.  Blows rcx, rdi
    // Superklass in rax.  Subklass in rbx.
!   // is "rbx <: rax" ? (value subclass <: array element superclass)
+   __ gen_subtype_check(rbx, ok_is_subtype, false);
  
    // Come here on failure
    // object is at TOS
    __ jump(ExternalAddress(Interpreter::_throw_ArrayStoreException_entry));
  

*** 1164,15 ***
    do_oop_store(_masm, element_address, rax, IS_ARRAY);
    __ jmp(done);
  
    // Have a null in rax, rdx=array, ecx=index.  Store null at ary[idx]
    __ bind(is_null);
!   __ profile_null_seen(rbx);
  
    // Store a null
    do_oop_store(_masm, element_address, noreg, IS_ARRAY);
  
    // Pop stack arguments
    __ bind(done);
    __ addptr(rsp, 3 * Interpreter::stackElementSize);
  }
  
--- 1195,59 ---
    do_oop_store(_masm, element_address, rax, IS_ARRAY);
    __ jmp(done);
  
    // Have a null in rax, rdx=array, ecx=index.  Store null at ary[idx]
    __ bind(is_null);
!   if (EnableValhalla) {
+     Label is_null_into_value_array_npe, store_null;
+ 
+     // No way to store null in null-free array
+     __ test_null_free_array_oop(rdx, rbx, is_null_into_value_array_npe);
+     __ jmp(store_null);
  
+     __ bind(is_null_into_value_array_npe);
+     __ jump(ExternalAddress(Interpreter::_throw_NullPointerException_entry));
+ 
+     __ bind(store_null);
+   }
    // Store a null
    do_oop_store(_masm, element_address, noreg, IS_ARRAY);
+   __ jmp(done);
+ 
+   if (UseFlatArray) {
+     Label is_type_ok;
+     __ bind(is_flat_array); // Store non-null value to flat
  
+     // Simplistic type check...
+ 
+     // Profile the not-null value's klass.
+     __ load_klass(rbx, rax, rscratch1);
+     // Move element klass into rax
+     __ movptr(rax, Address(rdi, ArrayKlass::element_klass_offset()));
+     // flat value array needs exact type match
+     // is "rax == rbx" (value subclass == array element superclass)
+     __ cmpptr(rax, rbx);
+     __ jccb(Assembler::equal, is_type_ok);
+ 
+     __ jump(ExternalAddress(Interpreter::_throw_ArrayStoreException_entry));
+ 
+     __ bind(is_type_ok);
+     // rbx: value's klass
+     // rdx: array
+     // rdi: array klass
+     __ test_klass_is_empty_inline_type(rbx, rax, done);
+ 
+     // calc dst for copy
+     __ movl(rax, at_tos_p1()); // index
+     __ data_for_value_array_index(rdx, rdi, rax, rax);
+ 
+     // ...and src for copy
+     __ movptr(rcx, at_tos());  // value
+     __ data_for_oop(rcx, rcx, rbx);
+ 
+     __ access_value_copy(IN_HEAP, rcx, rax, rbx);
+   }
    // Pop stack arguments
    __ bind(done);
    __ addptr(rsp, 3 * Interpreter::stackElementSize);
  }
  

*** 2337,17 ***
  }
  
  void TemplateTable::if_acmp(Condition cc) {
    transition(atos, vtos);
    // assume branch is more often taken than not (loops use backward branches)
!   Label not_taken;
    __ pop_ptr(rdx);
    __ cmpoop(rdx, rax);
    __ jcc(j_not(cc), not_taken);
    branch(false, false);
    __ bind(not_taken);
!   __ profile_not_taken_branch(rax);
  }
  
  void TemplateTable::ret() {
    transition(vtos, vtos);
    locals_index(rbx);
--- 2412,63 ---
  }
  
  void TemplateTable::if_acmp(Condition cc) {
    transition(atos, vtos);
    // assume branch is more often taken than not (loops use backward branches)
!   Label taken, not_taken;
    __ pop_ptr(rdx);
+ 
+   __ profile_acmp(rbx, rdx, rax, rcx);
+ 
+   const int is_inline_type_mask = markWord::inline_type_pattern;
+   if (EnableValhalla) {
+     __ cmpoop(rdx, rax);
+     __ jcc(Assembler::equal, (cc == equal) ? taken : not_taken);
+ 
+     // might be substitutable, test if either rax or rdx is null
+     __ testptr(rax, rax);
+     __ jcc(Assembler::zero, (cc == equal) ? not_taken : taken);
+     __ testptr(rdx, rdx);
+     __ jcc(Assembler::zero, (cc == equal) ? not_taken : taken);
+ 
+     // and both are values ?
+     __ movptr(rbx, Address(rdx, oopDesc::mark_offset_in_bytes()));
+     __ andptr(rbx, Address(rax, oopDesc::mark_offset_in_bytes()));
+     __ andptr(rbx, is_inline_type_mask);
+     __ cmpptr(rbx, is_inline_type_mask);
+     __ jcc(Assembler::notEqual, (cc == equal) ? not_taken : taken);
+ 
+     // same value klass ?
+     __ load_metadata(rbx, rdx);
+     __ load_metadata(rcx, rax);
+     __ cmpptr(rbx, rcx);
+     __ jcc(Assembler::notEqual, (cc == equal) ? not_taken : taken);
+ 
+     // Know both are the same type, let's test for substitutability...
+     if (cc == equal) {
+       invoke_is_substitutable(rax, rdx, taken, not_taken);
+     } else {
+       invoke_is_substitutable(rax, rdx, not_taken, taken);
+     }
+     __ stop("Not reachable");
+   }
+ 
    __ cmpoop(rdx, rax);
    __ jcc(j_not(cc), not_taken);
+   __ bind(taken);
    branch(false, false);
    __ bind(not_taken);
!   __ profile_not_taken_branch(rax, true);
+ }
+ 
+ void TemplateTable::invoke_is_substitutable(Register aobj, Register bobj,
+                                             Label& is_subst, Label& not_subst) {
+   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::is_substitutable), aobj, bobj);
+   // Restored...rax answer, jmp to outcome...
+   __ testl(rax, rax);
+   __ jcc(Assembler::zero, not_subst);
+   __ jmp(is_subst);
  }
  
  void TemplateTable::ret() {
    transition(vtos, vtos);
    locals_index(rbx);

*** 2613,11 ***
    // Need to narrow in the return bytecode rather than in generate_return_entry
    // since compiled code callers expect the result to already be narrowed.
    if (state == itos) {
      __ narrow(rax);
    }
!   __ remove_activation(state, rbcp);
  
    __ jmp(rbcp);
  }
  
  // ----------------------------------------------------------------------------
--- 2734,12 ---
    // Need to narrow in the return bytecode rather than in generate_return_entry
    // since compiled code callers expect the result to already be narrowed.
    if (state == itos) {
      __ narrow(rax);
    }
! 
+   __ remove_activation(state, rbcp, true, true, true);
  
    __ jmp(rbcp);
  }
  
  // ----------------------------------------------------------------------------

*** 2979,11 ***
  }
  
  void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
    transition(vtos, vtos);
  
!   const Register obj   = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
    const Register cache = rcx;
    const Register index = rdx;
    const Register off   = rbx;
    const Register tos_state   = rax;
    const Register flags = rdx;
--- 3101,11 ---
  }
  
  void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
    transition(vtos, vtos);
  
!   const Register obj   = LP64_ONLY(r9) NOT_LP64(rcx);
    const Register cache = rcx;
    const Register index = rdx;
    const Register off   = rbx;
    const Register tos_state   = rax;
    const Register flags = rdx;

*** 2991,22 ***
  
    resolve_cache_and_index_for_field(byte_no, cache, index);
    jvmti_post_field_access(cache, index, is_static, false);
    load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
  
-   if (!is_static) pop_and_check_object(obj);
- 
    const Address field(obj, off, Address::times_1, 0*wordSize);
  
!   Label Done, notByte, notBool, notInt, notShort, notChar, notLong, notFloat, notObj;
  
    // Make sure we don't need to mask edx after the above shift
    assert(btos == 0, "change code, btos != 0");
    __ testl(tos_state, tos_state);
    __ jcc(Assembler::notZero, notByte);
  
    // btos
    __ access_load_at(T_BYTE, IN_HEAP, rax, field, noreg, noreg);
    __ push(btos);
    // Rewrite bytecode to be faster
    if (!is_static && rc == may_rewrite) {
      patch_bytecode(Bytecodes::_fast_bgetfield, bc, rbx);
--- 3113,21 ---
  
    resolve_cache_and_index_for_field(byte_no, cache, index);
    jvmti_post_field_access(cache, index, is_static, false);
    load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
  
    const Address field(obj, off, Address::times_1, 0*wordSize);
  
!   Label Done, notByte, notBool, notInt, notShort, notChar, notLong, notFloat, notObj, notInlineType;
  
    // Make sure we don't need to mask edx after the above shift
    assert(btos == 0, "change code, btos != 0");
    __ testl(tos_state, tos_state);
    __ jcc(Assembler::notZero, notByte);
  
    // btos
+   if (!is_static) pop_and_check_object(obj);
    __ access_load_at(T_BYTE, IN_HEAP, rax, field, noreg, noreg);
    __ push(btos);
    // Rewrite bytecode to be faster
    if (!is_static && rc == may_rewrite) {
      patch_bytecode(Bytecodes::_fast_bgetfield, bc, rbx);

*** 3014,11 ***
    __ jmp(Done);
  
    __ bind(notByte);
    __ cmpl(tos_state, ztos);
    __ jcc(Assembler::notEqual, notBool);
! 
    // ztos (same code as btos)
    __ access_load_at(T_BOOLEAN, IN_HEAP, rax, field, noreg, noreg);
    __ push(ztos);
    // Rewrite bytecode to be faster
    if (!is_static && rc == may_rewrite) {
--- 3135,11 ---
    __ jmp(Done);
  
    __ bind(notByte);
    __ cmpl(tos_state, ztos);
    __ jcc(Assembler::notEqual, notBool);
!    if (!is_static) pop_and_check_object(obj);
    // ztos (same code as btos)
    __ access_load_at(T_BOOLEAN, IN_HEAP, rax, field, noreg, noreg);
    __ push(ztos);
    // Rewrite bytecode to be faster
    if (!is_static && rc == may_rewrite) {

*** 3029,18 ***
  
    __ bind(notBool);
    __ cmpl(tos_state, atos);
    __ jcc(Assembler::notEqual, notObj);
    // atos
!   do_oop_load(_masm, field, rax);
!   __ push(atos);
!   if (!is_static && rc == may_rewrite) {
!     patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx);
    }
-   __ jmp(Done);
  
    __ bind(notObj);
    __ cmpl(tos_state, itos);
    __ jcc(Assembler::notEqual, notInt);
    // itos
    __ access_load_at(T_INT, IN_HEAP, rax, field, noreg, noreg);
    __ push(itos);
--- 3150,96 ---
  
    __ bind(notBool);
    __ cmpl(tos_state, atos);
    __ jcc(Assembler::notEqual, notObj);
    // atos
!   if (!EnableValhalla) {
!     if (!is_static) pop_and_check_object(obj);
!     do_oop_load(_masm, field, rax);
!     __ push(atos);
+     if (!is_static && rc == may_rewrite) {
+       patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx);
+     }
+     __ jmp(Done);
+   } else {
+     if (is_static) {
+       __ load_heap_oop(rax, field);
+       Label is_null_free_inline_type, uninitialized;
+       // Issue below if the static field has not been initialized yet
+       __ test_field_is_null_free_inline_type(flags, rscratch1, is_null_free_inline_type);
+         // field is not a null free inline type
+         __ push(atos);
+         __ jmp(Done);
+       // field is a null free inline type, must not return null even if uninitialized
+       __ bind(is_null_free_inline_type);
+           __ testptr(rax, rax);
+         __ jcc(Assembler::zero, uninitialized);
+           __ push(atos);
+           __ jmp(Done);
+         __ bind(uninitialized);
+ #ifdef _LP64
+           Label slow_case, finish;
+           __ movptr(rbx, Address(obj, java_lang_Class::klass_offset()));
+           __ cmpb(Address(rbx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
+           __ jcc(Assembler::notEqual, slow_case);
+         __ get_default_value_oop(rbx, rscratch1, rax);
+         __ jmp(finish);
+         __ bind(slow_case);
+ #endif // LP64
+           __ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_static_inline_type_field),
+                 obj, cache);
+ #ifdef _LP64
+           __ bind(finish);
+   #endif // _LP64
+         __ verify_oop(rax);
+         __ push(atos);
+         __ jmp(Done);
+     } else {
+       Label is_flat, nonnull, is_inline_type, rewrite_inline;
+       __ test_field_is_null_free_inline_type(flags, rscratch1, is_inline_type);
+       // field is not a null free inline type
+       pop_and_check_object(obj);
+       __ load_heap_oop(rax, field);
+       __ push(atos);
+       if (rc == may_rewrite) {
+         patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx);
+       }
+       __ jmp(Done);
+       __ bind(is_inline_type);
+       __ test_field_is_flat(flags, rscratch1, is_flat);
+           // field is not flat
+           pop_and_check_object(obj);
+           __ load_heap_oop(rax, field);
+           __ testptr(rax, rax);
+           __ jcc(Assembler::notZero, nonnull);
+             __ load_unsigned_short(flags, Address(cache, in_bytes(ResolvedFieldEntry::field_index_offset())));
+             __ movptr(rcx, Address(cache, ResolvedFieldEntry::field_holder_offset()));
+             __ get_inline_type_field_klass(rcx, flags, rbx);
+             __ get_default_value_oop(rbx, rcx, rax);
+           __ bind(nonnull);
+           __ verify_oop(rax);
+           __ push(atos);
+           __ jmp(rewrite_inline);
+         __ bind(is_flat);
+           pop_and_check_object(rax);
+           __ load_unsigned_short(flags, Address(cache, in_bytes(ResolvedFieldEntry::field_index_offset())));
+           __ movptr(rcx, Address(cache, ResolvedFieldEntry::field_holder_offset()));
+           __ read_flat_field(rcx, flags, rbx, rax);
+           __ verify_oop(rax);
+           __ push(atos);
+       __ bind(rewrite_inline);
+       if (rc == may_rewrite) {
+         patch_bytecode(Bytecodes::_fast_qgetfield, bc, rbx);
+       }
+         __ jmp(Done);
+     }
    }
  
    __ bind(notObj);
+ 
+   if (!is_static) pop_and_check_object(obj);
+ 
    __ cmpl(tos_state, itos);
    __ jcc(Assembler::notEqual, notInt);
    // itos
    __ access_load_at(T_INT, IN_HEAP, rax, field, noreg, noreg);
    __ push(itos);

*** 3136,11 ***
  
  void TemplateTable::getstatic(int byte_no) {
    getfield_or_static(byte_no, true);
  }
  
- 
  // The registers cache and index expected to be set before call.
  // The function may destroy various registers, just not the cache and index registers.
  void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
    // Cache is rcx and index is rdx
    const Register entry = LP64_ONLY(c_rarg2) NOT_LP64(rax); // ResolvedFieldEntry
--- 3335,10 ---

*** 3218,11 ***
    const Register obj = rcx;
    const Register cache = rcx;
    const Register index = rdx;
    const Register tos_state   = rdx;
    const Register off   = rbx;
!   const Register flags = rax;
  
    resolve_cache_and_index_for_field(byte_no, cache, index);
    jvmti_post_field_mod(cache, index, is_static);
    load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
  
--- 3416,11 ---
    const Register obj = rcx;
    const Register cache = rcx;
    const Register index = rdx;
    const Register tos_state   = rdx;
    const Register off   = rbx;
!   const Register flags = r9;
  
    resolve_cache_and_index_for_field(byte_no, cache, index);
    jvmti_post_field_mod(cache, index, is_static);
    load_resolved_field_entry(obj, cache, tos_state, off, flags, is_static);
  

*** 3231,34 ***
    //                                              Assembler::StoreStore));
  
    Label notVolatile, Done;
  
    // Check for volatile store
!   __ andl(flags, (1 << ResolvedFieldEntry::is_volatile_shift));
!   __ testl(flags, flags);
    __ jcc(Assembler::zero, notVolatile);
  
!   putfield_or_static_helper(byte_no, is_static, rc, obj, off, tos_state);
    volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
                                                 Assembler::StoreStore));
    __ jmp(Done);
    __ bind(notVolatile);
  
!   putfield_or_static_helper(byte_no, is_static, rc, obj, off, tos_state);
  
    __ bind(Done);
  }
  
  void TemplateTable::putfield_or_static_helper(int byte_no, bool is_static, RewriteControl rc,
!                                               Register obj, Register off, Register tos_state) {
  
    // field addresses
    const Address field(obj, off, Address::times_1, 0*wordSize);
    NOT_LP64( const Address hi(obj, off, Address::times_1, 1*wordSize);)
  
    Label notByte, notBool, notInt, notShort, notChar,
!         notLong, notFloat, notObj;
    Label Done;
  
    const Register bc    = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
  
    // Test TOS state
--- 3429,35 ---
    //                                              Assembler::StoreStore));
  
    Label notVolatile, Done;
  
    // Check for volatile store
!   __ movl(rscratch1, flags);
!   __ andl(rscratch1, (1 << ResolvedFieldEntry::is_volatile_shift));
+   __ testl(rscratch1, rscratch1);
    __ jcc(Assembler::zero, notVolatile);
  
!   putfield_or_static_helper(byte_no, is_static, rc, obj, off, tos_state, flags);
    volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
                                                 Assembler::StoreStore));
    __ jmp(Done);
    __ bind(notVolatile);
  
!   putfield_or_static_helper(byte_no, is_static, rc, obj, off, tos_state, flags);
  
    __ bind(Done);
  }
  
  void TemplateTable::putfield_or_static_helper(int byte_no, bool is_static, RewriteControl rc,
!                                               Register obj, Register off, Register tos_state, Register flags) {
  
    // field addresses
    const Address field(obj, off, Address::times_1, 0*wordSize);
    NOT_LP64( const Address hi(obj, off, Address::times_1, 1*wordSize);)
  
    Label notByte, notBool, notInt, notShort, notChar,
!         notLong, notFloat, notObj, notInlineType;
    Label Done;
  
    const Register bc    = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
  
    // Test TOS state

*** 3295,18 ***
    __ cmpl(tos_state, atos);
    __ jcc(Assembler::notEqual, notObj);
  
    // atos
    {
!     __ pop(atos);
!     if (!is_static) pop_and_check_object(obj);
!     // Store into the field
!     do_oop_store(_masm, field, rax);
!     if (!is_static && rc == may_rewrite) {
!       patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no);
      }
-     __ jmp(Done);
    }
  
    __ bind(notObj);
    __ cmpl(tos_state, itos);
    __ jcc(Assembler::notEqual, notInt);
--- 3494,64 ---
    __ cmpl(tos_state, atos);
    __ jcc(Assembler::notEqual, notObj);
  
    // atos
    {
!     if (!EnableValhalla) {
!       __ pop(atos);
!       if (!is_static) pop_and_check_object(obj);
!       // Store into the field
!       do_oop_store(_masm, field, rax);
!       if (!is_static && rc == may_rewrite) {
+         patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no);
+       }
+       __ jmp(Done);
+     } else {
+       __ pop(atos);
+       if (is_static) {
+         Label is_inline_type;
+         __ test_field_is_not_null_free_inline_type(flags, rscratch1, is_inline_type);
+         __ null_check(rax);
+         __ bind(is_inline_type);
+         do_oop_store(_masm, field, rax);
+         __ jmp(Done);
+       } else {
+         Label is_inline_type, is_flat, rewrite_not_inline, rewrite_inline;
+         __ test_field_is_null_free_inline_type(flags, rscratch1, is_inline_type);
+         // Not an inline type
+         pop_and_check_object(obj);
+         // Store into the field
+         do_oop_store(_masm, field, rax);
+         __ bind(rewrite_not_inline);
+         if (rc == may_rewrite) {
+           patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no);
+         }
+         __ jmp(Done);
+         // Implementation of the inline type semantic
+         __ bind(is_inline_type);
+         __ null_check(rax);
+         __ test_field_is_flat(flags, rscratch1, is_flat);
+         // field is not flat
+         pop_and_check_object(obj);
+         // Store into the field
+         do_oop_store(_masm, field, rax);
+         __ jmp(rewrite_inline);
+         __ bind(is_flat);
+         // field is flat
+         pop_and_check_object(obj);
+         assert_different_registers(rax, rdx, obj, off);
+         __ load_klass(rdx, rax, rscratch1);
+         __ data_for_oop(rax, rax, rdx);
+         __ addptr(obj, off);
+         __ access_value_copy(IN_HEAP, rax, obj, rdx);
+         __ bind(rewrite_inline);
+         if (rc == may_rewrite) {
+           patch_bytecode(Bytecodes::_fast_qputfield, bc, rbx, true, byte_no);
+         }
+         __ jmp(Done);
+       }
      }
    }
  
    __ bind(notObj);
    __ cmpl(tos_state, itos);
    __ jcc(Assembler::notEqual, notInt);

*** 3441,10 ***
--- 3686,11 ---
      __ push_ptr(rbx);                 // put the object pointer back on tos
      // Save tos values before call_VM() clobbers them. Since we have
      // to do it for every data type, we use the saved values as the
      // jvalue object.
      switch (bytecode()) {          // load values into the jvalue object
+     case Bytecodes::_fast_qputfield: //fall through
      case Bytecodes::_fast_aputfield: __ push_ptr(rax); break;
      case Bytecodes::_fast_bputfield: // fall through
      case Bytecodes::_fast_zputfield: // fall through
      case Bytecodes::_fast_sputfield: // fall through
      case Bytecodes::_fast_cputfield: // fall through

*** 3466,10 ***
--- 3712,11 ---
      // c_rarg3: jvalue object on the stack
      LP64_ONLY(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification), rbx, c_rarg2, c_rarg3));
      NOT_LP64(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification), rbx, rax, rcx));
  
      switch (bytecode()) {             // restore tos values
+     case Bytecodes::_fast_qputfield: // fall through
      case Bytecodes::_fast_aputfield: __ pop_ptr(rax); break;
      case Bytecodes::_fast_bputfield: // fall through
      case Bytecodes::_fast_zputfield: // fall through
      case Bytecodes::_fast_sputfield: // fall through
      case Bytecodes::_fast_cputfield: // fall through

*** 3493,41 ***
    jvmti_post_fast_field_mod();
  
    __ push(rax);
    __ load_field_entry(rcx, rax);
    load_resolved_field_entry(noreg, cache, rax, rbx, rdx);
-   // RBX: field offset, RCX: RAX: TOS, RDX: flags
-   __ andl(rdx, (1 << ResolvedFieldEntry::is_volatile_shift));
    __ pop(rax);
  
    // Get object from stack
    pop_and_check_object(rcx);
  
    // field address
    const Address field(rcx, rbx, Address::times_1);
  
    // Check for volatile store
!   __ testl(rdx, rdx);
    __ jcc(Assembler::zero, notVolatile);
  
!   fast_storefield_helper(field, rax);
    volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
                                                 Assembler::StoreStore));
    __ jmp(Done);
    __ bind(notVolatile);
  
!   fast_storefield_helper(field, rax);
  
    __ bind(Done);
  }
  
! void TemplateTable::fast_storefield_helper(Address field, Register rax) {
  
    // access field
    switch (bytecode()) {
    case Bytecodes::_fast_aputfield:
!     do_oop_store(_masm, field, rax);
      break;
    case Bytecodes::_fast_lputfield:
  #ifdef _LP64
      __ access_store_at(T_LONG, IN_HEAP, field, noreg /* ltos */, noreg, noreg, noreg);
  #else
--- 3740,61 ---
    jvmti_post_fast_field_mod();
  
    __ push(rax);
    __ load_field_entry(rcx, rax);
    load_resolved_field_entry(noreg, cache, rax, rbx, rdx);
    __ pop(rax);
+   // RBX: field offset, RCX: RAX: TOS, RDX: flags
  
    // Get object from stack
    pop_and_check_object(rcx);
  
    // field address
    const Address field(rcx, rbx, Address::times_1);
  
    // Check for volatile store
!   __ movl(rscratch2, rdx);  // saving flags for is_flat test
+   __ andl(rscratch2, (1 << ResolvedFieldEntry::is_volatile_shift));
+   __ testl(rscratch2, rscratch2);
    __ jcc(Assembler::zero, notVolatile);
  
!   fast_storefield_helper(field, rax, rdx);
    volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
                                                 Assembler::StoreStore));
    __ jmp(Done);
    __ bind(notVolatile);
  
!   fast_storefield_helper(field, rax, rdx);
  
    __ bind(Done);
  }
  
! void TemplateTable::fast_storefield_helper(Address field, Register rax, Register flags) {
  
    // access field
    switch (bytecode()) {
+   case Bytecodes::_fast_qputfield:
+     {
+       Label is_flat, done;
+       __ null_check(rax);
+       __ test_field_is_flat(flags, rscratch1, is_flat);
+       // field is not flat
+       do_oop_store(_masm, field, rax);
+       __ jmp(done);
+       __ bind(is_flat);
+       // field is flat
+       __ load_klass(rdx, rax, rscratch1);
+       __ data_for_oop(rax, rax, rdx);
+       __ lea(rcx, field);
+       __ access_value_copy(IN_HEAP, rax, rcx, rdx);
+       __ bind(done);
+     }
+     break;
    case Bytecodes::_fast_aputfield:
!     {
+       do_oop_store(_masm, field, rax);
+     }
      break;
    case Bytecodes::_fast_lputfield:
  #ifdef _LP64
      __ access_store_at(T_LONG, IN_HEAP, field, noreg /* ltos */, noreg, noreg, noreg);
  #else

*** 3585,19 ***
      __ bind(L1);
    }
  
    // access constant pool cache
    __ load_field_entry(rcx, rbx);
!   __ load_sized_value(rbx, Address(rcx, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
  
    // rax: object
    __ verify_oop(rax);
    __ null_check(rax);
!   Address field(rax, rbx, Address::times_1);
  
    // access field
    switch (bytecode()) {
    case Bytecodes::_fast_agetfield:
      do_oop_load(_masm, field, rax);
      __ verify_oop(rax);
      break;
    case Bytecodes::_fast_lgetfield:
--- 3852,46 ---
      __ bind(L1);
    }
  
    // access constant pool cache
    __ load_field_entry(rcx, rbx);
!   __ load_sized_value(rdx, Address(rcx, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
  
    // rax: object
    __ verify_oop(rax);
    __ null_check(rax);
!   Address field(rax, rdx, Address::times_1);
  
    // access field
    switch (bytecode()) {
+   case Bytecodes::_fast_qgetfield:
+     {
+       Label is_flat, nonnull, Done;
+       __ load_unsigned_byte(rscratch1, Address(rcx, in_bytes(ResolvedFieldEntry::flags_offset())));
+       __ test_field_is_flat(rscratch1, rscratch2, is_flat);
+         // field is not flat
+         __ load_heap_oop(rax, field);
+         __ testptr(rax, rax);
+         __ jcc(Assembler::notZero, nonnull);
+           __ load_unsigned_short(rdx, Address(rcx, in_bytes(ResolvedFieldEntry::field_index_offset())));
+           __ movptr(rcx, Address(rcx, ResolvedFieldEntry::field_holder_offset()));
+           __ get_inline_type_field_klass(rcx, rdx, rbx);
+           __ get_default_value_oop(rbx, rcx, rax);
+         __ bind(nonnull);
+         __ verify_oop(rax);
+         __ jmp(Done);
+       __ bind(is_flat);
+       // field is flat
+         __ push(rdx); // save offset
+         __ load_unsigned_short(rdx, Address(rcx, in_bytes(ResolvedFieldEntry::field_index_offset())));
+         __ movptr(rcx, Address(rcx, ResolvedFieldEntry::field_holder_offset()));
+         __ pop(rbx); // restore offset
+         __ read_flat_field(rcx, rdx, rbx, rax);
+       __ bind(Done);
+       __ verify_oop(rax);
+     }
+     break;
    case Bytecodes::_fast_agetfield:
      do_oop_load(_masm, field, rax);
      __ verify_oop(rax);
      break;
    case Bytecodes::_fast_lgetfield:

*** 4029,114 ***
  
  void TemplateTable::_new() {
    transition(vtos, atos);
    __ get_unsigned_2_byte_index_at_bcp(rdx, 1);
    Label slow_case;
-   Label slow_case_no_pop;
    Label done;
-   Label initialize_header;
  
    __ get_cpool_and_tags(rcx, rax);
  
    // Make sure the class we're about to instantiate has been resolved.
    // This is done before loading InstanceKlass to be consistent with the order
    // how Constant Pool is updated (see ConstantPool::klass_at_put)
    const int tags_offset = Array<u1>::base_offset_in_bytes();
    __ cmpb(Address(rax, rdx, Address::times_1, tags_offset), JVM_CONSTANT_Class);
!   __ jcc(Assembler::notEqual, slow_case_no_pop);
  
    // get InstanceKlass
    __ load_resolved_klass_at_index(rcx, rcx, rdx);
-   __ push(rcx);  // save the contexts of klass for initializing the header
  
    // make sure klass is initialized & doesn't have finalizer
-   // make sure klass is fully initialized
    __ cmpb(Address(rcx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
    __ jcc(Assembler::notEqual, slow_case);
  
!   // get instance_size in InstanceKlass (scaled to a count of bytes)
!   __ movl(rdx, Address(rcx, Klass::layout_helper_offset()));
-   // test to see if it has a finalizer or is malformed in some way
-   __ testl(rdx, Klass::_lh_instance_slow_path_bit);
-   __ jcc(Assembler::notZero, slow_case);
- 
-   // Allocate the instance:
-   //  If TLAB is enabled:
-   //    Try to allocate in the TLAB.
-   //    If fails, go to the slow path.
-   //    Initialize the allocation.
-   //    Exit.
-   //
-   //  Go to slow path.
- 
-   const Register thread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
- 
-   if (UseTLAB) {
-     NOT_LP64(__ get_thread(thread);)
-     __ tlab_allocate(thread, rax, rdx, 0, rcx, rbx, slow_case);
-     if (ZeroTLAB) {
-       // the fields have been already cleared
-       __ jmp(initialize_header);
-     }
- 
-     // The object is initialized before the header.  If the object size is
-     // zero, go directly to the header initialization.
-     __ decrement(rdx, sizeof(oopDesc));
-     __ jcc(Assembler::zero, initialize_header);
- 
-     // Initialize topmost object field, divide rdx by 8, check if odd and
-     // test if zero.
-     __ xorl(rcx, rcx);    // use zero reg to clear memory (shorter code)
-     __ shrl(rdx, LogBytesPerLong); // divide by 2*oopSize and set carry flag if odd
- 
-     // rdx must have been multiple of 8
- #ifdef ASSERT
-     // make sure rdx was multiple of 8
-     Label L;
-     // Ignore partial flag stall after shrl() since it is debug VM
-     __ jcc(Assembler::carryClear, L);
-     __ stop("object size is not multiple of 2 - adjust this code");
-     __ bind(L);
-     // rdx must be > 0, no extra check needed here
- #endif
- 
-     // initialize remaining object fields: rdx was a multiple of 8
-     { Label loop;
-     __ bind(loop);
-     __ movptr(Address(rax, rdx, Address::times_8, sizeof(oopDesc) - 1*oopSize), rcx);
-     NOT_LP64(__ movptr(Address(rax, rdx, Address::times_8, sizeof(oopDesc) - 2*oopSize), rcx));
-     __ decrement(rdx);
-     __ jcc(Assembler::notZero, loop);
-     }
- 
-     // initialize object header only.
-     __ bind(initialize_header);
-     __ movptr(Address(rax, oopDesc::mark_offset_in_bytes()),
-               (intptr_t)markWord::prototype().value()); // header
-     __ pop(rcx);   // get saved klass back in the register.
- #ifdef _LP64
-     __ xorl(rsi, rsi); // use zero reg to clear memory (shorter code)
-     __ store_klass_gap(rax, rsi);  // zero klass gap for compressed oops
- #endif
-     __ store_klass(rax, rcx, rscratch1);  // klass
- 
-     {
-       SkipIfEqual skip_if(_masm, &DTraceAllocProbes, 0, rscratch1);
-       // Trigger dtrace event for fastpath
-       __ push(atos);
-       __ call_VM_leaf(
-            CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), rax);
-       __ pop(atos);
-     }
- 
-     __ jmp(done);
-   }
  
    // slow case
    __ bind(slow_case);
-   __ pop(rcx);   // restore stack pointer to what it was when we came in.
-   __ bind(slow_case_no_pop);
  
    Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rax);
    Register rarg2 = LP64_ONLY(c_rarg2) NOT_LP64(rdx);
  
    __ get_constant_pool(rarg1);
--- 4323,33 ---
  
  void TemplateTable::_new() {
    transition(vtos, atos);
    __ get_unsigned_2_byte_index_at_bcp(rdx, 1);
    Label slow_case;
    Label done;
  
    __ get_cpool_and_tags(rcx, rax);
  
    // Make sure the class we're about to instantiate has been resolved.
    // This is done before loading InstanceKlass to be consistent with the order
    // how Constant Pool is updated (see ConstantPool::klass_at_put)
    const int tags_offset = Array<u1>::base_offset_in_bytes();
    __ cmpb(Address(rax, rdx, Address::times_1, tags_offset), JVM_CONSTANT_Class);
!   __ jcc(Assembler::notEqual, slow_case);
  
    // get InstanceKlass
    __ load_resolved_klass_at_index(rcx, rcx, rdx);
  
    // make sure klass is initialized & doesn't have finalizer
    __ cmpb(Address(rcx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
    __ jcc(Assembler::notEqual, slow_case);
  
!   __ allocate_instance(rcx, rax, rdx, rbx, true, slow_case);
!   __ jmp(done);
  
    // slow case
    __ bind(slow_case);
  
    Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rax);
    Register rarg2 = LP64_ONLY(c_rarg2) NOT_LP64(rdx);
  
    __ get_constant_pool(rarg1);

*** 4181,14 ***
  
    // Get cpool & tags index
    __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
    __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
    // See if bytecode has already been quicked
!   __ cmpb(Address(rdx, rbx,
!                   Address::times_1,
!                   Array<u1>::base_offset_in_bytes()),
!           JVM_CONSTANT_Class);
    __ jcc(Assembler::equal, quicked);
    __ push(atos); // save receiver for result, and for GC
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
  
    // vm_result_2 has metadata result
--- 4394,14 ---
  
    // Get cpool & tags index
    __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
    __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
    // See if bytecode has already been quicked
!   __ movzbl(rdx, Address(rdx, rbx,
!       Address::times_1,
!       Array<u1>::base_offset_in_bytes()));
!   __ cmpl(rdx, JVM_CONSTANT_Class);
    __ jcc(Assembler::equal, quicked);
    __ push(atos); // save receiver for result, and for GC
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
  
    // vm_result_2 has metadata result

*** 4222,19 ***
    __ jump(ExternalAddress(Interpreter::_throw_ClassCastException_entry));
  
    // Come here on success
    __ bind(ok_is_subtype);
    __ mov(rax, rdx); // Restore object in rdx
  
    // Collect counts on whether this check-cast sees nulls a lot or not.
    if (ProfileInterpreter) {
-     __ jmp(done);
-     __ bind(is_null);
      __ profile_null_seen(rcx);
-   } else {
-     __ bind(is_null);   // same as 'done'
    }
    __ bind(done);
  }
  
  void TemplateTable::instanceof() {
    transition(atos, itos);
--- 4435,19 ---
    __ jump(ExternalAddress(Interpreter::_throw_ClassCastException_entry));
  
    // Come here on success
    __ bind(ok_is_subtype);
    __ mov(rax, rdx); // Restore object in rdx
+   __ jmp(done);
+ 
+   __ bind(is_null);
  
    // Collect counts on whether this check-cast sees nulls a lot or not.
    if (ProfileInterpreter) {
      __ profile_null_seen(rcx);
    }
+ 
    __ bind(done);
  }
  
  void TemplateTable::instanceof() {
    transition(atos, itos);

*** 4244,14 ***
  
    // Get cpool & tags index
    __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
    __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
    // See if bytecode has already been quicked
!   __ cmpb(Address(rdx, rbx,
!                   Address::times_1,
!                   Array<u1>::base_offset_in_bytes()),
!           JVM_CONSTANT_Class);
    __ jcc(Assembler::equal, quicked);
  
    __ push(atos); // save receiver for result, and for GC
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
    // vm_result_2 has metadata result
--- 4457,14 ---
  
    // Get cpool & tags index
    __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
    __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
    // See if bytecode has already been quicked
!   __ movzbl(rdx, Address(rdx, rbx,
!         Address::times_1,
!         Array<u1>::base_offset_in_bytes()));
!   __ cmpl(rdx, JVM_CONSTANT_Class);
    __ jcc(Assembler::equal, quicked);
  
    __ push(atos); // save receiver for result, and for GC
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
    // vm_result_2 has metadata result

*** 4299,11 ***
    __ bind(done);
    // rax = 0: obj == nullptr or  obj is not an instanceof the specified klass
    // rax = 1: obj != nullptr and obj is     an instanceof the specified klass
  }
  
- 
  //----------------------------------------------------------------------------------------------------
  // Breakpoints
  void TemplateTable::_breakpoint() {
    // Note: We get here even if we are single stepping..
    // jbug insists on setting breakpoints at every bytecode
--- 4512,10 ---

*** 4361,10 ***
--- 4573,14 ---
    transition(atos, vtos);
  
    // check for null object
    __ null_check(rax);
  
+   Label is_inline_type;
+   __ movptr(rbx, Address(rax, oopDesc::mark_offset_in_bytes()));
+   __ test_markword_is_inline_type(rbx, is_inline_type);
+ 
    const Address monitor_block_top(
          rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
    const Address monitor_block_bot(
          rbp, frame::interpreter_frame_initial_sp_offset * wordSize);
    const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();

*** 4453,18 ***
--- 4669,34 ---
    __ generate_stack_overflow_check(0);
  
    // The bcp has already been incremented. Just need to dispatch to
    // next instruction.
    __ dispatch_next(vtos);
+ 
+   __ bind(is_inline_type);
+   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
+                     InterpreterRuntime::throw_illegal_monitor_state_exception));
+   __ should_not_reach_here();
  }
  
  void TemplateTable::monitorexit() {
    transition(atos, vtos);
  
    // check for null object
    __ null_check(rax);
  
+   const int is_inline_type_mask = markWord::inline_type_pattern;
+   Label has_identity;
+   __ movptr(rbx, Address(rax, oopDesc::mark_offset_in_bytes()));
+   __ andptr(rbx, is_inline_type_mask);
+   __ cmpl(rbx, is_inline_type_mask);
+   __ jcc(Assembler::notEqual, has_identity);
+   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
+                      InterpreterRuntime::throw_illegal_monitor_state_exception));
+   __ should_not_reach_here();
+   __ bind(has_identity);
+ 
    const Address monitor_block_top(
          rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
    const Address monitor_block_bot(
          rbp, frame::interpreter_frame_initial_sp_offset * wordSize);
    const int entry_size = frame::interpreter_frame_monitor_size_in_bytes();
< prev index next >