< prev index next >

src/hotspot/cpu/x86/templateTable_x86.cpp

Print this page
*** 33,10 ***
--- 33,11 ---
  #include "interpreter/templateTable.hpp"
  #include "memory/universe.hpp"
  #include "oops/methodData.hpp"
  #include "oops/objArrayKlass.hpp"
  #include "oops/oop.inline.hpp"
+ #include "oops/inlineKlass.hpp"
  #include "prims/jvmtiExport.hpp"
  #include "prims/methodHandles.hpp"
  #include "runtime/frame.inline.hpp"
  #include "runtime/safepointMechanism.hpp"
  #include "runtime/sharedRuntime.hpp"

*** 150,11 ***
  static void do_oop_store(InterpreterMacroAssembler* _masm,
                           Address dst,
                           Register val,
                           DecoratorSet decorators = 0) {
    assert(val == noreg || val == rax, "parameter is just for looks");
!   __ store_heap_oop(dst, val, rdx, rbx, decorators);
  }
  
  static void do_oop_load(InterpreterMacroAssembler* _masm,
                          Address src,
                          Register dst,
--- 151,11 ---
  static void do_oop_store(InterpreterMacroAssembler* _masm,
                           Address dst,
                           Register val,
                           DecoratorSet decorators = 0) {
    assert(val == noreg || val == rax, "parameter is just for looks");
!   __ store_heap_oop(dst, val, rdx, rbx, noreg, decorators);
  }
  
  static void do_oop_load(InterpreterMacroAssembler* _masm,
                          Address src,
                          Register dst,

*** 173,10 ***
--- 174,11 ---
                                     int byte_no) {
    if (!RewriteBytecodes)  return;
    Label L_patch_done;
  
    switch (bc) {
+   case Bytecodes::_fast_qputfield:
    case Bytecodes::_fast_aputfield:
    case Bytecodes::_fast_bputfield:
    case Bytecodes::_fast_zputfield:
    case Bytecodes::_fast_cputfield:
    case Bytecodes::_fast_dputfield:

*** 365,10 ***
--- 367,11 ---
    const int base_offset = ConstantPool::header_size() * wordSize;
    const int tags_offset = Array<u1>::base_offset_in_bytes();
  
    // get type
    __ movzbl(rdx, Address(rax, rbx, Address::times_1, tags_offset));
+   __ andl(rdx, ~JVM_CONSTANT_QDescBit);
  
    // unresolved class - get the resolved class
    __ cmpl(rdx, JVM_CONSTANT_UnresolvedClass);
    __ jccb(Assembler::equal, call_ldc);
  

*** 816,19 ***
                      noreg, noreg);
  }
  
  void TemplateTable::aaload() {
    transition(itos, atos);
!   // rax: index
!   // rdx: array
!   index_check(rdx, rax); // kills rbx
!   do_oop_load(_masm,
!               Address(rdx, rax,
!                       UseCompressedOops ? Address::times_4 : Address::times_ptr,
!                       arrayOopDesc::base_offset_in_bytes(T_OBJECT)),
!               rax,
!               IS_ARRAY);
  }
  
  void TemplateTable::baload() {
    transition(itos, itos);
    // rax: index
--- 819,37 ---
                      noreg, noreg);
  }
  
  void TemplateTable::aaload() {
    transition(itos, atos);
!   Register array = rdx;
!   Register index = rax;
! 
!   index_check(array, index); // kills rbx
!   __ profile_array(rbx, array, rcx);
!   if (UseFlatArray) {
!     Label is_flat_array, done;
!     __ test_flattened_array_oop(array, rbx, is_flat_array);
!     do_oop_load(_masm,
+                 Address(array, index,
+                         UseCompressedOops ? Address::times_4 : Address::times_ptr,
+                         arrayOopDesc::base_offset_in_bytes(T_OBJECT)),
+                 rax,
+                 IS_ARRAY);
+     __ jmp(done);
+     __ bind(is_flat_array);
+     __ read_flattened_element(array, index, rbx, rcx, rax);
+     __ bind(done);
+   } else {
+     do_oop_load(_masm,
+                 Address(array, index,
+                         UseCompressedOops ? Address::times_4 : Address::times_ptr,
+                         arrayOopDesc::base_offset_in_bytes(T_OBJECT)),
+                 rax,
+                 IS_ARRAY);
+   }
+   __ profile_element(rbx, rax, rcx);
  }
  
  void TemplateTable::baload() {
    transition(itos, itos);
    // rax: index

*** 1110,11 ***
                               arrayOopDesc::base_offset_in_bytes(T_DOUBLE)),
                       noreg /* dtos */, noreg, noreg);
  }
  
  void TemplateTable::aastore() {
!   Label is_null, ok_is_subtype, done;
    transition(vtos, vtos);
    // stack: ..., array, index, value
    __ movptr(rax, at_tos());    // value
    __ movl(rcx, at_tos_p1()); // index
    __ movptr(rdx, at_tos_p2()); // array
--- 1131,11 ---
                               arrayOopDesc::base_offset_in_bytes(T_DOUBLE)),
                       noreg /* dtos */, noreg, noreg);
  }
  
  void TemplateTable::aastore() {
!   Label is_null, is_flat_array, ok_is_subtype, done;
    transition(vtos, vtos);
    // stack: ..., array, index, value
    __ movptr(rax, at_tos());    // value
    __ movl(rcx, at_tos_p1()); // index
    __ movptr(rdx, at_tos_p2()); // array

*** 1122,24 ***
    Address element_address(rdx, rcx,
                            UseCompressedOops? Address::times_4 : Address::times_ptr,
                            arrayOopDesc::base_offset_in_bytes(T_OBJECT));
  
    index_check_without_pop(rdx, rcx);     // kills rbx
    __ testptr(rax, rax);
    __ jcc(Assembler::zero, is_null);
  
    Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
    // Move subklass into rbx
    __ load_klass(rbx, rax, tmp_load_klass);
!   // Move superklass into rax
!   __ load_klass(rax, rdx, tmp_load_klass);
-   __ movptr(rax, Address(rax,
                           ObjArrayKlass::element_klass_offset()));
  
    // Generate subtype check.  Blows rcx, rdi
    // Superklass in rax.  Subklass in rbx.
!   __ gen_subtype_check(rbx, ok_is_subtype);
  
    // Come here on failure
    // object is at TOS
    __ jump(ExternalAddress(Interpreter::_throw_ArrayStoreException_entry));
  
--- 1143,35 ---
    Address element_address(rdx, rcx,
                            UseCompressedOops? Address::times_4 : Address::times_ptr,
                            arrayOopDesc::base_offset_in_bytes(T_OBJECT));
  
    index_check_without_pop(rdx, rcx);     // kills rbx
+ 
+   __ profile_array(rdi, rdx, rbx);
+   __ profile_element(rdi, rax, rbx);
+ 
    __ testptr(rax, rax);
    __ jcc(Assembler::zero, is_null);
  
+   // Move array class to rdi
    Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
+   __ load_klass(rdi, rdx, tmp_load_klass);
+   if (UseFlatArray) {
+     __ movl(rbx, Address(rdi, Klass::layout_helper_offset()));
+     __ test_flattened_array_layout(rbx, is_flat_array);
+   }
+ 
    // Move subklass into rbx
    __ load_klass(rbx, rax, tmp_load_klass);
!   // Move array element superklass into rax
!   __ movptr(rax, Address(rdi,
                           ObjArrayKlass::element_klass_offset()));
  
    // Generate subtype check.  Blows rcx, rdi
    // Superklass in rax.  Subklass in rbx.
!   // is "rbx <: rax" ? (value subclass <: array element superclass)
+   __ gen_subtype_check(rbx, ok_is_subtype, false);
  
    // Come here on failure
    // object is at TOS
    __ jump(ExternalAddress(Interpreter::_throw_ArrayStoreException_entry));
  

*** 1153,15 ***
    do_oop_store(_masm, element_address, rax, IS_ARRAY);
    __ jmp(done);
  
    // Have a NULL in rax, rdx=array, ecx=index.  Store NULL at ary[idx]
    __ bind(is_null);
!   __ profile_null_seen(rbx);
  
    // Store a NULL
    do_oop_store(_masm, element_address, noreg, IS_ARRAY);
  
    // Pop stack arguments
    __ bind(done);
    __ addptr(rsp, 3 * Interpreter::stackElementSize);
  }
  
--- 1185,59 ---
    do_oop_store(_masm, element_address, rax, IS_ARRAY);
    __ jmp(done);
  
    // Have a NULL in rax, rdx=array, ecx=index.  Store NULL at ary[idx]
    __ bind(is_null);
!   if (EnableValhalla) {
+     Label is_null_into_value_array_npe, store_null;
+ 
+     // No way to store null in null-free array
+     __ test_null_free_array_oop(rdx, rbx, is_null_into_value_array_npe);
+     __ jmp(store_null);
  
+     __ bind(is_null_into_value_array_npe);
+     __ jump(ExternalAddress(Interpreter::_throw_NullPointerException_entry));
+ 
+     __ bind(store_null);
+   }
    // Store a NULL
    do_oop_store(_masm, element_address, noreg, IS_ARRAY);
+   __ jmp(done);
+ 
+   if (EnableValhalla) {
+     Label is_type_ok;
+     __ bind(is_flat_array); // Store non-null value to flat
+ 
+     // Simplistic type check...
+ 
+     // Profile the not-null value's klass.
+     __ load_klass(rbx, rax, tmp_load_klass);
+     // Move element klass into rax
+     __ movptr(rax, Address(rdi, ArrayKlass::element_klass_offset()));
+     // flat value array needs exact type match
+     // is "rax == rbx" (value subclass == array element superclass)
+     __ cmpptr(rax, rbx);
+     __ jccb(Assembler::equal, is_type_ok);
  
+     __ jump(ExternalAddress(Interpreter::_throw_ArrayStoreException_entry));
+ 
+     __ bind(is_type_ok);
+     // rbx: value's klass
+     // rdx: array
+     // rdi: array klass
+     __ test_klass_is_empty_inline_type(rbx, rax, done);
+ 
+     // calc dst for copy
+     __ movl(rax, at_tos_p1()); // index
+     __ data_for_value_array_index(rdx, rdi, rax, rax);
+ 
+     // ...and src for copy
+     __ movptr(rcx, at_tos());  // value
+     __ data_for_oop(rcx, rcx, rbx);
+ 
+     __ access_value_copy(IN_HEAP, rcx, rax, rbx);
+   }
    // Pop stack arguments
    __ bind(done);
    __ addptr(rsp, 3 * Interpreter::stackElementSize);
  }
  

*** 2328,17 ***
  }
  
  void TemplateTable::if_acmp(Condition cc) {
    transition(atos, vtos);
    // assume branch is more often taken than not (loops use backward branches)
!   Label not_taken;
    __ pop_ptr(rdx);
    __ cmpoop(rdx, rax);
    __ jcc(j_not(cc), not_taken);
    branch(false, false);
    __ bind(not_taken);
!   __ profile_not_taken_branch(rax);
  }
  
  void TemplateTable::ret() {
    transition(vtos, vtos);
    locals_index(rbx);
--- 2404,63 ---
  }
  
  void TemplateTable::if_acmp(Condition cc) {
    transition(atos, vtos);
    // assume branch is more often taken than not (loops use backward branches)
!   Label taken, not_taken;
    __ pop_ptr(rdx);
+ 
+   __ profile_acmp(rbx, rdx, rax, rcx);
+ 
+   const int is_inline_type_mask = markWord::inline_type_pattern;
+   if (EnableValhalla) {
+     __ cmpoop(rdx, rax);
+     __ jcc(Assembler::equal, (cc == equal) ? taken : not_taken);
+ 
+     // might be substitutable, test if either rax or rdx is null
+     __ testptr(rax, rax);
+     __ jcc(Assembler::zero, (cc == equal) ? not_taken : taken);
+     __ testptr(rdx, rdx);
+     __ jcc(Assembler::zero, (cc == equal) ? not_taken : taken);
+ 
+     // and both are values ?
+     __ movptr(rbx, Address(rdx, oopDesc::mark_offset_in_bytes()));
+     __ andptr(rbx, Address(rax, oopDesc::mark_offset_in_bytes()));
+     __ andptr(rbx, is_inline_type_mask);
+     __ cmpptr(rbx, is_inline_type_mask);
+     __ jcc(Assembler::notEqual, (cc == equal) ? not_taken : taken);
+ 
+     // same value klass ?
+     __ load_metadata(rbx, rdx);
+     __ load_metadata(rcx, rax);
+     __ cmpptr(rbx, rcx);
+     __ jcc(Assembler::notEqual, (cc == equal) ? not_taken : taken);
+ 
+     // Know both are the same type, let's test for substitutability...
+     if (cc == equal) {
+       invoke_is_substitutable(rax, rdx, taken, not_taken);
+     } else {
+       invoke_is_substitutable(rax, rdx, not_taken, taken);
+     }
+     __ stop("Not reachable");
+   }
+ 
    __ cmpoop(rdx, rax);
    __ jcc(j_not(cc), not_taken);
+   __ bind(taken);
    branch(false, false);
    __ bind(not_taken);
!   __ profile_not_taken_branch(rax, true);
+ }
+ 
+ void TemplateTable::invoke_is_substitutable(Register aobj, Register bobj,
+                                             Label& is_subst, Label& not_subst) {
+   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::is_substitutable), aobj, bobj);
+   // Restored...rax answer, jmp to outcome...
+   __ testl(rax, rax);
+   __ jcc(Assembler::zero, not_subst);
+   __ jmp(is_subst);
  }
  
  void TemplateTable::ret() {
    transition(vtos, vtos);
    locals_index(rbx);

*** 2603,11 ***
    // Need to narrow in the return bytecode rather than in generate_return_entry
    // since compiled code callers expect the result to already be narrowed.
    if (state == itos) {
      __ narrow(rax);
    }
!   __ remove_activation(state, rbcp);
  
    __ jmp(rbcp);
  }
  
  // ----------------------------------------------------------------------------
--- 2725,12 ---
    // Need to narrow in the return bytecode rather than in generate_return_entry
    // since compiled code callers expect the result to already be narrowed.
    if (state == itos) {
      __ narrow(rax);
    }
! 
+   __ remove_activation(state, rbcp, true, true, true);
  
    __ jmp(rbcp);
  }
  
  // ----------------------------------------------------------------------------

*** 2801,41 ***
    const Register index = rdx;
    const Register obj   = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
    const Register off   = rbx;
    const Register flags = rax;
    const Register bc    = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // uses same reg as obj, so don't mix them
  
    resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
    jvmti_post_field_access(cache, index, is_static, false);
    load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
  
-   if (!is_static) pop_and_check_object(obj);
- 
    const Address field(obj, off, Address::times_1, 0*wordSize);
  
!   Label Done, notByte, notBool, notInt, notShort, notChar, notLong, notFloat, notObj;
  
    __ shrl(flags, ConstantPoolCacheEntry::tos_state_shift);
    // Make sure we don't need to mask edx after the above shift
    assert(btos == 0, "change code, btos != 0");
  
    __ andl(flags, ConstantPoolCacheEntry::tos_state_mask);
  
    __ jcc(Assembler::notZero, notByte);
    // btos
    __ access_load_at(T_BYTE, IN_HEAP, rax, field, noreg, noreg);
    __ push(btos);
    // Rewrite bytecode to be faster
    if (!is_static && rc == may_rewrite) {
      patch_bytecode(Bytecodes::_fast_bgetfield, bc, rbx);
    }
    __ jmp(Done);
  
    __ bind(notByte);
    __ cmpl(flags, ztos);
    __ jcc(Assembler::notEqual, notBool);
! 
    // ztos (same code as btos)
    __ access_load_at(T_BOOLEAN, IN_HEAP, rax, field, noreg, noreg);
    __ push(ztos);
    // Rewrite bytecode to be faster
    if (!is_static && rc == may_rewrite) {
--- 2924,50 ---
    const Register index = rdx;
    const Register obj   = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
    const Register off   = rbx;
    const Register flags = rax;
    const Register bc    = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // uses same reg as obj, so don't mix them
+   const Register flags2 = rdx;
  
    resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
    jvmti_post_field_access(cache, index, is_static, false);
    load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
  
    const Address field(obj, off, Address::times_1, 0*wordSize);
  
!   Label Done, notByte, notBool, notInt, notShort, notChar, notLong, notFloat, notObj, notInlineType;
+ 
+   if (!is_static) {
+     __ movptr(rcx, Address(cache, index, Address::times_ptr,
+                            in_bytes(ConstantPoolCache::base_offset() +
+                                     ConstantPoolCacheEntry::f1_offset())));
+   }
+ 
+   __ movl(flags2, flags);
  
    __ shrl(flags, ConstantPoolCacheEntry::tos_state_shift);
    // Make sure we don't need to mask edx after the above shift
    assert(btos == 0, "change code, btos != 0");
  
    __ andl(flags, ConstantPoolCacheEntry::tos_state_mask);
  
    __ jcc(Assembler::notZero, notByte);
    // btos
+   if (!is_static) pop_and_check_object(obj);
    __ access_load_at(T_BYTE, IN_HEAP, rax, field, noreg, noreg);
    __ push(btos);
    // Rewrite bytecode to be faster
    if (!is_static && rc == may_rewrite) {
      patch_bytecode(Bytecodes::_fast_bgetfield, bc, rbx);
    }
    __ jmp(Done);
  
    __ bind(notByte);
+ 
    __ cmpl(flags, ztos);
    __ jcc(Assembler::notEqual, notBool);
!    if (!is_static) pop_and_check_object(obj);
    // ztos (same code as btos)
    __ access_load_at(T_BOOLEAN, IN_HEAP, rax, field, noreg, noreg);
    __ push(ztos);
    // Rewrite bytecode to be faster
    if (!is_static && rc == may_rewrite) {

*** 2846,18 ***
  
    __ bind(notBool);
    __ cmpl(flags, atos);
    __ jcc(Assembler::notEqual, notObj);
    // atos
!   do_oop_load(_masm, field, rax);
!   __ push(atos);
!   if (!is_static && rc == may_rewrite) {
!     patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx);
    }
-   __ jmp(Done);
  
    __ bind(notObj);
    __ cmpl(flags, itos);
    __ jcc(Assembler::notEqual, notInt);
    // itos
    __ access_load_at(T_INT, IN_HEAP, rax, field, noreg, noreg);
    __ push(itos);
--- 2978,104 ---
  
    __ bind(notBool);
    __ cmpl(flags, atos);
    __ jcc(Assembler::notEqual, notObj);
    // atos
!   if (!EnableValhalla) {
!     if (!is_static) pop_and_check_object(obj);
!     do_oop_load(_masm, field, rax);
!     __ push(atos);
+     if (!is_static && rc == may_rewrite) {
+       patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx);
+     }
+     __ jmp(Done);
+   } else {
+     if (is_static) {
+       __ load_heap_oop(rax, field);
+       if (EnableValhalla) {
+         Label is_null_free_inline_type, uninitialized;
+         // Issue below if the static field has not been initialized yet
+         __ test_field_is_null_free_inline_type(flags2, rscratch1, is_null_free_inline_type);
+           // field is not a null free inline type
+           __ push(atos);
+           __ jmp(Done);
+         // field is a null free inline type, must not return null even if uninitialized
+         __ bind(is_null_free_inline_type);
+            __ testptr(rax, rax);
+           __ jcc(Assembler::zero, uninitialized);
+             __ push(atos);
+             __ jmp(Done);
+           __ bind(uninitialized);
+             __ andl(flags2, ConstantPoolCacheEntry::field_index_mask);
+   #ifdef _LP64
+             Label slow_case, finish;
+             __ cmpb(Address(rcx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
+             __ jcc(Assembler::notEqual, slow_case);
+           __ get_default_value_oop(rcx, off, rax);
+           __ jmp(finish);
+           __ bind(slow_case);
+   #endif // LP64
+             __ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::uninitialized_static_inline_type_field),
+                   obj, flags2);
+   #ifdef _LP64
+             __ bind(finish);
+   #endif // _LP64
+       }
+         __ verify_oop(rax);
+         __ push(atos);
+         __ jmp(Done);
+     } else {
+       Label is_inlined, nonnull, is_inline_type, rewrite_inline;
+       if (EnableValhalla) {
+         __ test_field_is_null_free_inline_type(flags2, rscratch1, is_inline_type);
+       }
+       // field is not a null free inline type
+       pop_and_check_object(obj);
+       __ load_heap_oop(rax, field);
+       __ push(atos);
+       if (rc == may_rewrite) {
+         patch_bytecode(Bytecodes::_fast_agetfield, bc, rbx);
+       }
+       __ jmp(Done);
+       if (EnableValhalla) {
+         __ bind(is_inline_type);
+           __ test_field_is_inlined(flags2, rscratch1, is_inlined);
+             // field is not inlined
+             __ movptr(rax, rcx);  // small dance required to preserve the klass_holder somewhere
+             pop_and_check_object(obj);
+             __ push(rax);
+             __ load_heap_oop(rax, field);
+             __ pop(rcx);
+             __ testptr(rax, rax);
+             __ jcc(Assembler::notZero, nonnull);
+               __ andl(flags2, ConstantPoolCacheEntry::field_index_mask);
+               __ get_inline_type_field_klass(rcx, flags2, rbx);
+               __ get_default_value_oop(rbx, rcx, rax);
+             __ bind(nonnull);
+             __ verify_oop(rax);
+             __ push(atos);
+             __ jmp(rewrite_inline);
+           __ bind(is_inlined);
+           // field is inlined
+             __ andl(flags2, ConstantPoolCacheEntry::field_index_mask);
+             pop_and_check_object(rax);
+             __ read_inlined_field(rcx, flags2, rbx, rax);
+             __ verify_oop(rax);
+             __ push(atos);
+         __ bind(rewrite_inline);
+         if (rc == may_rewrite) {
+           patch_bytecode(Bytecodes::_fast_qgetfield, bc, rbx);
+         }
+         __ jmp(Done);
+       }
+     }
    }
  
    __ bind(notObj);
+ 
+   if (!is_static) pop_and_check_object(obj);
+ 
    __ cmpl(flags, itos);
    __ jcc(Assembler::notEqual, notInt);
    // itos
    __ access_load_at(T_INT, IN_HEAP, rax, field, noreg, noreg);
    __ push(itos);

*** 2953,10 ***
--- 3171,32 ---
  
  void TemplateTable::getstatic(int byte_no) {
    getfield_or_static(byte_no, true);
  }
  
+ void TemplateTable::withfield() {
+   transition(vtos, atos);
+ 
+   Register cache = LP64_ONLY(c_rarg1) NOT_LP64(rcx);
+   Register index = LP64_ONLY(c_rarg2) NOT_LP64(rdx);
+ 
+   resolve_cache_and_index(f2_byte, cache, index, sizeof(u2));
+ 
+   Register cpentry = rbx;
+ 
+   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
+ 
+   __ lea(cpentry, Address(cache, index, Address::times_ptr,
+                          in_bytes(cp_base_offset)));
+   __ lea(rax, at_tos());
+   __ call_VM(rbx, CAST_FROM_FN_PTR(address, InterpreterRuntime::withfield), cpentry, rax);
+   // new value type is returned in rbx
+   // stack adjustment is returned in rax
+   __ verify_oop(rbx);
+   __ addptr(rsp, rax);
+   __ movptr(rax, rbx);
+ }
  
  // The registers cache and index expected to be set before call.
  // The function may destroy various registers, just not the cache and index registers.
  void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
  

*** 3048,10 ***
--- 3288,11 ---
    const Register cache = rcx;
    const Register index = rdx;
    const Register obj   = rcx;
    const Register off   = rbx;
    const Register flags = rax;
+   const Register flags2 = rdx;
  
    resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
    jvmti_post_field_mod(cache, index, is_static);
    load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
  

*** 3064,32 ***
    __ shrl(rdx, ConstantPoolCacheEntry::is_volatile_shift);
    __ andl(rdx, 0x1);
  
    // Check for volatile store
    __ testl(rdx, rdx);
    __ jcc(Assembler::zero, notVolatile);
  
!   putfield_or_static_helper(byte_no, is_static, rc, obj, off, flags);
    volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
                                                 Assembler::StoreStore));
    __ jmp(Done);
    __ bind(notVolatile);
  
!   putfield_or_static_helper(byte_no, is_static, rc, obj, off, flags);
  
    __ bind(Done);
  }
  
  void TemplateTable::putfield_or_static_helper(int byte_no, bool is_static, RewriteControl rc,
!                                               Register obj, Register off, Register flags) {
  
    // field addresses
    const Address field(obj, off, Address::times_1, 0*wordSize);
    NOT_LP64( const Address hi(obj, off, Address::times_1, 1*wordSize);)
  
    Label notByte, notBool, notInt, notShort, notChar,
!         notLong, notFloat, notObj;
    Label Done;
  
    const Register bc    = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
  
    __ shrl(flags, ConstantPoolCacheEntry::tos_state_shift);
--- 3305,33 ---
    __ shrl(rdx, ConstantPoolCacheEntry::is_volatile_shift);
    __ andl(rdx, 0x1);
  
    // Check for volatile store
    __ testl(rdx, rdx);
+   __ movl(flags2, flags);
    __ jcc(Assembler::zero, notVolatile);
  
!   putfield_or_static_helper(byte_no, is_static, rc, obj, off, flags, flags2);
    volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
                                                 Assembler::StoreStore));
    __ jmp(Done);
    __ bind(notVolatile);
  
!   putfield_or_static_helper(byte_no, is_static, rc, obj, off, flags, flags2);
  
    __ bind(Done);
  }
  
  void TemplateTable::putfield_or_static_helper(int byte_no, bool is_static, RewriteControl rc,
!                                               Register obj, Register off, Register flags, Register flags2) {
  
    // field addresses
    const Address field(obj, off, Address::times_1, 0*wordSize);
    NOT_LP64( const Address hi(obj, off, Address::times_1, 1*wordSize);)
  
    Label notByte, notBool, notInt, notShort, notChar,
!         notLong, notFloat, notObj, notInlineType;
    Label Done;
  
    const Register bc    = LP64_ONLY(c_rarg3) NOT_LP64(rcx);
  
    __ shrl(flags, ConstantPoolCacheEntry::tos_state_shift);

*** 3128,18 ***
    __ cmpl(flags, atos);
    __ jcc(Assembler::notEqual, notObj);
  
    // atos
    {
!     __ pop(atos);
!     if (!is_static) pop_and_check_object(obj);
!     // Store into the field
!     do_oop_store(_masm, field, rax);
!     if (!is_static && rc == may_rewrite) {
!       patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no);
      }
-     __ jmp(Done);
    }
  
    __ bind(notObj);
    __ cmpl(flags, itos);
    __ jcc(Assembler::notEqual, notInt);
--- 3370,70 ---
    __ cmpl(flags, atos);
    __ jcc(Assembler::notEqual, notObj);
  
    // atos
    {
!     if (!EnableValhalla) {
!       __ pop(atos);
!       if (!is_static) pop_and_check_object(obj);
!       // Store into the field
!       do_oop_store(_masm, field, rax);
!       if (!is_static && rc == may_rewrite) {
+         patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no);
+       }
+       __ jmp(Done);
+     } else {
+       __ pop(atos);
+       if (is_static) {
+         Label is_inline_type;
+         if (EnableValhalla) {
+           __ test_field_is_not_null_free_inline_type(flags2, rscratch1, is_inline_type);
+           __ null_check(rax);
+           __ bind(is_inline_type);
+         }
+         do_oop_store(_masm, field, rax);
+         __ jmp(Done);
+       } else {
+         Label is_inline_type, is_inlined, rewrite_not_inline, rewrite_inline;
+         if (EnableValhalla) {
+           __ test_field_is_null_free_inline_type(flags2, rscratch1, is_inline_type);
+         }
+         // Not an inline type
+         pop_and_check_object(obj);
+         // Store into the field
+         do_oop_store(_masm, field, rax);
+         __ bind(rewrite_not_inline);
+         if (rc == may_rewrite) {
+           patch_bytecode(Bytecodes::_fast_aputfield, bc, rbx, true, byte_no);
+         }
+         __ jmp(Done);
+         if (EnableValhalla) {
+           // Implementation of the inline type semantic
+           __ bind(is_inline_type);
+           __ null_check(rax);
+           __ test_field_is_inlined(flags2, rscratch1, is_inlined);
+           // field is not inlined
+           pop_and_check_object(obj);
+           // Store into the field
+           do_oop_store(_masm, field, rax);
+           __ jmp(rewrite_inline);
+           __ bind(is_inlined);
+           // field is inlined
+           pop_and_check_object(obj);
+           assert_different_registers(rax, rdx, obj, off);
+           __ load_klass(rdx, rax, rscratch1);
+           __ data_for_oop(rax, rax, rdx);
+           __ addptr(obj, off);
+           __ access_value_copy(IN_HEAP, rax, obj, rdx);
+           __ bind(rewrite_inline);
+           if (rc == may_rewrite) {
+             patch_bytecode(Bytecodes::_fast_qputfield, bc, rbx, true, byte_no);
+           }
+           __ jmp(Done);
+         }
+       }
      }
    }
  
    __ bind(notObj);
    __ cmpl(flags, itos);
    __ jcc(Assembler::notEqual, notInt);

*** 3274,10 ***
--- 3568,11 ---
      __ push_ptr(rbx);                 // put the object pointer back on tos
      // Save tos values before call_VM() clobbers them. Since we have
      // to do it for every data type, we use the saved values as the
      // jvalue object.
      switch (bytecode()) {          // load values into the jvalue object
+     case Bytecodes::_fast_qputfield: //fall through
      case Bytecodes::_fast_aputfield: __ push_ptr(rax); break;
      case Bytecodes::_fast_bputfield: // fall through
      case Bytecodes::_fast_zputfield: // fall through
      case Bytecodes::_fast_sputfield: // fall through
      case Bytecodes::_fast_cputfield: // fall through

*** 3299,10 ***
--- 3594,11 ---
      // c_rarg3: jvalue object on the stack
      LP64_ONLY(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification), rbx, c_rarg2, c_rarg3));
      NOT_LP64(__ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification), rbx, rax, rcx));
  
      switch (bytecode()) {             // restore tos values
+     case Bytecodes::_fast_qputfield: // fall through
      case Bytecodes::_fast_aputfield: __ pop_ptr(rax); break;
      case Bytecodes::_fast_bputfield: // fall through
      case Bytecodes::_fast_zputfield: // fall through
      case Bytecodes::_fast_sputfield: // fall through
      case Bytecodes::_fast_cputfield: // fall through

*** 3338,10 ***
--- 3634,14 ---
    // [jk] not needed currently
    // volatile_barrier(Assembler::Membar_mask_bits(Assembler::LoadStore |
    //                                              Assembler::StoreStore));
  
    Label notVolatile, Done;
+   if (bytecode() == Bytecodes::_fast_qputfield) {
+     __ movl(rscratch2, rdx);  // saving flags for is_inlined test
+   }
+ 
    __ shrl(rdx, ConstantPoolCacheEntry::is_volatile_shift);
    __ andl(rdx, 0x1);
  
    // Get object from stack
    pop_and_check_object(rcx);

*** 3351,27 ***
  
    // Check for volatile store
    __ testl(rdx, rdx);
    __ jcc(Assembler::zero, notVolatile);
  
!   fast_storefield_helper(field, rax);
    volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
                                                 Assembler::StoreStore));
    __ jmp(Done);
    __ bind(notVolatile);
  
!   fast_storefield_helper(field, rax);
  
    __ bind(Done);
  }
  
! void TemplateTable::fast_storefield_helper(Address field, Register rax) {
  
    // access field
    switch (bytecode()) {
    case Bytecodes::_fast_aputfield:
!     do_oop_store(_masm, field, rax);
      break;
    case Bytecodes::_fast_lputfield:
  #ifdef _LP64
      __ access_store_at(T_LONG, IN_HEAP, field, noreg /* ltos */, noreg, noreg);
  #else
--- 3651,52 ---
  
    // Check for volatile store
    __ testl(rdx, rdx);
    __ jcc(Assembler::zero, notVolatile);
  
!   if (bytecode() == Bytecodes::_fast_qputfield) {
+     __ movl(rdx, rscratch2);  // restoring flags for is_inlined test
+   }
+   fast_storefield_helper(field, rax, rdx);
    volatile_barrier(Assembler::Membar_mask_bits(Assembler::StoreLoad |
                                                 Assembler::StoreStore));
    __ jmp(Done);
    __ bind(notVolatile);
  
!   if (bytecode() == Bytecodes::_fast_qputfield) {
+     __ movl(rdx, rscratch2);  // restoring flags for is_inlined test
+   }
+   fast_storefield_helper(field, rax, rdx);
  
    __ bind(Done);
  }
  
! void TemplateTable::fast_storefield_helper(Address field, Register rax, Register flags) {
  
    // access field
    switch (bytecode()) {
+   case Bytecodes::_fast_qputfield:
+     {
+       Label is_inlined, done;
+       __ null_check(rax);
+       __ test_field_is_inlined(flags, rscratch1, is_inlined);
+       // field is not inlined
+       do_oop_store(_masm, field, rax);
+       __ jmp(done);
+       __ bind(is_inlined);
+       // field is inlined
+       __ load_klass(rdx, rax, rscratch1);
+       __ data_for_oop(rax, rax, rdx);
+       __ lea(rcx, field);
+       __ access_value_copy(IN_HEAP, rax, rcx, rdx);
+       __ bind(done);
+     }
+     break;
    case Bytecodes::_fast_aputfield:
!     {
+       do_oop_store(_masm, field, rax);
+     }
      break;
    case Bytecodes::_fast_lputfield:
  #ifdef _LP64
      __ access_store_at(T_LONG, IN_HEAP, field, noreg /* ltos */, noreg, noreg);
  #else

*** 3437,21 ***
    //                      in_bytes(ConstantPoolCache::base_offset() +
    //                               ConstantPoolCacheEntry::flags_offset())));
    // __ shrl(rdx, ConstantPoolCacheEntry::is_volatile_shift);
    // __ andl(rdx, 0x1);
    //
!   __ movptr(rbx, Address(rcx, rbx, Address::times_ptr,
                           in_bytes(ConstantPoolCache::base_offset() +
                                    ConstantPoolCacheEntry::f2_offset())));
  
    // rax: object
    __ verify_oop(rax);
    __ null_check(rax);
!   Address field(rax, rbx, Address::times_1);
  
    // access field
    switch (bytecode()) {
    case Bytecodes::_fast_agetfield:
      do_oop_load(_masm, field, rax);
      __ verify_oop(rax);
      break;
    case Bytecodes::_fast_lgetfield:
--- 3762,60 ---
    //                      in_bytes(ConstantPoolCache::base_offset() +
    //                               ConstantPoolCacheEntry::flags_offset())));
    // __ shrl(rdx, ConstantPoolCacheEntry::is_volatile_shift);
    // __ andl(rdx, 0x1);
    //
!   __ movptr(rdx, Address(rcx, rbx, Address::times_ptr,
                           in_bytes(ConstantPoolCache::base_offset() +
                                    ConstantPoolCacheEntry::f2_offset())));
  
    // rax: object
    __ verify_oop(rax);
    __ null_check(rax);
!   Address field(rax, rdx, Address::times_1);
  
    // access field
    switch (bytecode()) {
+   case Bytecodes::_fast_qgetfield:
+     {
+       Label is_inlined, nonnull, Done;
+       __ movptr(rscratch1, Address(rcx, rbx, Address::times_ptr,
+                                    in_bytes(ConstantPoolCache::base_offset() +
+                                             ConstantPoolCacheEntry::flags_offset())));
+       __ test_field_is_inlined(rscratch1, rscratch2, is_inlined);
+         // field is not inlined
+         __ load_heap_oop(rax, field);
+         __ testptr(rax, rax);
+         __ jcc(Assembler::notZero, nonnull);
+           __ movl(rdx, Address(rcx, rbx, Address::times_ptr,
+                              in_bytes(ConstantPoolCache::base_offset() +
+                                       ConstantPoolCacheEntry::flags_offset())));
+           __ andl(rdx, ConstantPoolCacheEntry::field_index_mask);
+           __ movptr(rcx, Address(rcx, rbx, Address::times_ptr,
+                                        in_bytes(ConstantPoolCache::base_offset() +
+                                                 ConstantPoolCacheEntry::f1_offset())));
+           __ get_inline_type_field_klass(rcx, rdx, rbx);
+           __ get_default_value_oop(rbx, rcx, rax);
+         __ bind(nonnull);
+         __ verify_oop(rax);
+         __ jmp(Done);
+       __ bind(is_inlined);
+       // field is inlined
+         __ push(rdx); // save offset
+         __ movl(rdx, Address(rcx, rbx, Address::times_ptr,
+                            in_bytes(ConstantPoolCache::base_offset() +
+                                     ConstantPoolCacheEntry::flags_offset())));
+         __ andl(rdx, ConstantPoolCacheEntry::field_index_mask);
+         __ movptr(rcx, Address(rcx, rbx, Address::times_ptr,
+                                      in_bytes(ConstantPoolCache::base_offset() +
+                                               ConstantPoolCacheEntry::f1_offset())));
+         __ pop(rbx); // restore offset
+         __ read_inlined_field(rcx, rdx, rbx, rax);
+       __ bind(Done);
+       __ verify_oop(rax);
+     }
+     break;
    case Bytecodes::_fast_agetfield:
      do_oop_load(_masm, field, rax);
      __ verify_oop(rax);
      break;
    case Bytecodes::_fast_lgetfield:

*** 3916,152 ***
  
  void TemplateTable::_new() {
    transition(vtos, atos);
    __ get_unsigned_2_byte_index_at_bcp(rdx, 1);
    Label slow_case;
-   Label slow_case_no_pop;
    Label done;
!   Label initialize_header;
-   Label initialize_object;  // including clearing the fields
  
    __ get_cpool_and_tags(rcx, rax);
  
    // Make sure the class we're about to instantiate has been resolved.
    // This is done before loading InstanceKlass to be consistent with the order
    // how Constant Pool is updated (see ConstantPool::klass_at_put)
    const int tags_offset = Array<u1>::base_offset_in_bytes();
    __ cmpb(Address(rax, rdx, Address::times_1, tags_offset), JVM_CONSTANT_Class);
!   __ jcc(Assembler::notEqual, slow_case_no_pop);
  
    // get InstanceKlass
    __ load_resolved_klass_at_index(rcx, rcx, rdx);
!   __ push(rcx);  // save the contexts of klass for initializing the header
  
    // make sure klass is initialized & doesn't have finalizer
-   // make sure klass is fully initialized
    __ cmpb(Address(rcx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
    __ jcc(Assembler::notEqual, slow_case);
  
!   // get instance_size in InstanceKlass (scaled to a count of bytes)
!   __ movl(rdx, Address(rcx, Klass::layout_helper_offset()));
-   // test to see if it has a finalizer or is malformed in some way
-   __ testl(rdx, Klass::_lh_instance_slow_path_bit);
-   __ jcc(Assembler::notZero, slow_case);
- 
-   // Allocate the instance:
-   //  If TLAB is enabled:
-   //    Try to allocate in the TLAB.
-   //    If fails, go to the slow path.
-   //  Else If inline contiguous allocations are enabled:
-   //    Try to allocate in eden.
-   //    If fails due to heap end, go to slow path.
-   //
-   //  If TLAB is enabled OR inline contiguous is enabled:
-   //    Initialize the allocation.
-   //    Exit.
-   //
-   //  Go to slow path.
  
!   const bool allow_shared_alloc =
!     Universe::heap()->supports_inline_contig_alloc();
  
!   const Register thread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
! #ifndef _LP64
-   if (UseTLAB || allow_shared_alloc) {
-     __ get_thread(thread);
-   }
- #endif // _LP64
  
!   if (UseTLAB) {
!     __ tlab_allocate(thread, rax, rdx, 0, rcx, rbx, slow_case);
!     if (ZeroTLAB) {
!       // the fields have been already cleared
-       __ jmp(initialize_header);
-     } else {
-       // initialize both the header and fields
-       __ jmp(initialize_object);
-     }
-   } else {
-     // Allocation in the shared Eden, if allowed.
-     //
-     // rdx: instance size in bytes
-     __ eden_allocate(thread, rax, rdx, 0, rbx, slow_case);
-   }
  
!   // If UseTLAB or allow_shared_alloc are true, the object is created above and
!   // there is an initialize need. Otherwise, skip and go to the slow path.
!   if (UseTLAB || allow_shared_alloc) {
-     // The object is initialized before the header.  If the object size is
-     // zero, go directly to the header initialization.
-     __ bind(initialize_object);
-     __ decrement(rdx, sizeof(oopDesc));
-     __ jcc(Assembler::zero, initialize_header);
- 
-     // Initialize topmost object field, divide rdx by 8, check if odd and
-     // test if zero.
-     __ xorl(rcx, rcx);    // use zero reg to clear memory (shorter code)
-     __ shrl(rdx, LogBytesPerLong); // divide by 2*oopSize and set carry flag if odd
- 
-     // rdx must have been multiple of 8
- #ifdef ASSERT
-     // make sure rdx was multiple of 8
-     Label L;
-     // Ignore partial flag stall after shrl() since it is debug VM
-     __ jcc(Assembler::carryClear, L);
-     __ stop("object size is not multiple of 2 - adjust this code");
-     __ bind(L);
-     // rdx must be > 0, no extra check needed here
- #endif
  
!     // initialize remaining object fields: rdx was a multiple of 8
!     { Label loop;
-     __ bind(loop);
-     __ movptr(Address(rax, rdx, Address::times_8, sizeof(oopDesc) - 1*oopSize), rcx);
-     NOT_LP64(__ movptr(Address(rax, rdx, Address::times_8, sizeof(oopDesc) - 2*oopSize), rcx));
-     __ decrement(rdx);
-     __ jcc(Assembler::notZero, loop);
-     }
  
!     // initialize object header only.
!     __ bind(initialize_header);
!     __ movptr(Address(rax, oopDesc::mark_offset_in_bytes()),
-               (intptr_t)markWord::prototype().value()); // header
-     __ pop(rcx);   // get saved klass back in the register.
- #ifdef _LP64
-     __ xorl(rsi, rsi); // use zero reg to clear memory (shorter code)
-     __ store_klass_gap(rax, rsi);  // zero klass gap for compressed oops
- #endif
-     Register tmp_store_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
-     __ store_klass(rax, rcx, tmp_store_klass);  // klass
  
!     {
!       SkipIfEqual skip_if(_masm, &DTraceAllocProbes, 0);
-       // Trigger dtrace event for fastpath
-       __ push(atos);
-       __ call_VM_leaf(
-            CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), rax);
-       __ pop(atos);
-     }
  
!     __ jmp(done);
!   }
  
-   // slow case
    __ bind(slow_case);
-   __ pop(rcx);   // restore stack pointer to what it was when we came in.
-   __ bind(slow_case_no_pop);
  
!   Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rax);
    Register rarg2 = LP64_ONLY(c_rarg2) NOT_LP64(rdx);
  
-   __ get_constant_pool(rarg1);
    __ get_unsigned_2_byte_index_at_bcp(rarg2, 1);
!   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), rarg1, rarg2);
!    __ verify_oop(rax);
  
-   // continue
    __ bind(done);
  }
  
  void TemplateTable::newarray() {
    transition(itos, atos);
    Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rdx);
--- 4280,103 ---
  
  void TemplateTable::_new() {
    transition(vtos, atos);
    __ get_unsigned_2_byte_index_at_bcp(rdx, 1);
    Label slow_case;
    Label done;
!   Label is_not_value;
  
    __ get_cpool_and_tags(rcx, rax);
  
    // Make sure the class we're about to instantiate has been resolved.
    // This is done before loading InstanceKlass to be consistent with the order
    // how Constant Pool is updated (see ConstantPool::klass_at_put)
    const int tags_offset = Array<u1>::base_offset_in_bytes();
    __ cmpb(Address(rax, rdx, Address::times_1, tags_offset), JVM_CONSTANT_Class);
!   __ jcc(Assembler::notEqual, slow_case);
  
    // get InstanceKlass
    __ load_resolved_klass_at_index(rcx, rcx, rdx);
! 
+   __ cmpb(Address(rcx, InstanceKlass::kind_offset()), InstanceKlass::_kind_inline_type);
+   __ jcc(Assembler::notEqual, is_not_value);
+ 
+   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_InstantiationError));
+ 
+   __ bind(is_not_value);
  
    // make sure klass is initialized & doesn't have finalizer
    __ cmpb(Address(rcx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
    __ jcc(Assembler::notEqual, slow_case);
  
!   __ allocate_instance(rcx, rax, rdx, rbx, true, slow_case);
!   __ jmp(done);
  
!   // slow case
!   __ bind(slow_case);
  
!   Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rax);
!   Register rarg2 = LP64_ONLY(c_rarg2) NOT_LP64(rdx);
  
!   __ get_constant_pool(rarg1);
!   __ get_unsigned_2_byte_index_at_bcp(rarg2, 1);
!   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), rarg1, rarg2);
!    __ verify_oop(rax);
  
!   // continue
!   __ bind(done);
! }
  
! void TemplateTable::defaultvalue() {
!   transition(vtos, atos);
  
!   Label slow_case;
!   Label done;
!   Label is_value;
  
!   __ get_unsigned_2_byte_index_at_bcp(rdx, 1);
!   __ get_cpool_and_tags(rcx, rax);
  
!   // Make sure the class we're about to instantiate has been resolved.
!   // This is done before loading InstanceKlass to be consistent with the order
+   // how Constant Pool is updated (see ConstantPool::klass_at_put)
+   const int tags_offset = Array<u1>::base_offset_in_bytes();
+   __ cmpb(Address(rax, rdx, Address::times_1, tags_offset), JVM_CONSTANT_Class);
+   __ jcc(Assembler::notEqual, slow_case);
+ 
+   // get InstanceKlass
+   __ load_resolved_klass_at_index(rcx, rcx, rdx);
+ 
+   __ cmpb(Address(rcx, InstanceKlass::kind_offset()), InstanceKlass::_kind_inline_type);
+   __ jcc(Assembler::equal, is_value);
+ 
+   // in the future, defaultvalue will just return null instead of throwing an exception
+   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_IncompatibleClassChangeError));
+ 
+   __ bind(is_value);
+ 
+   // make sure klass is fully initialized
+   __ cmpb(Address(rcx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
+   __ jcc(Assembler::notEqual, slow_case);
+ 
+   // have a resolved InlineKlass in rcx, return the default value oop from it
+   __ get_default_value_oop(rcx, rdx, rax);
+   __ jmp(done);
  
    __ bind(slow_case);
  
!   Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rcx);
    Register rarg2 = LP64_ONLY(c_rarg2) NOT_LP64(rdx);
  
    __ get_unsigned_2_byte_index_at_bcp(rarg2, 1);
!   __ get_constant_pool(rarg1);
! 
+   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::defaultvalue),
+       rarg1, rarg2);
  
    __ bind(done);
+   __ verify_oop(rax);
  }
  
  void TemplateTable::newarray() {
    transition(itos, atos);
    Register rarg1 = LP64_ONLY(c_rarg1) NOT_LP64(rdx);

*** 4096,14 ***
  
    // Get cpool & tags index
    __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
    __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
    // See if bytecode has already been quicked
!   __ cmpb(Address(rdx, rbx,
!                   Address::times_1,
!                   Array<u1>::base_offset_in_bytes()),
!           JVM_CONSTANT_Class);
    __ jcc(Assembler::equal, quicked);
    __ push(atos); // save receiver for result, and for GC
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
  
    // vm_result_2 has metadata result
--- 4411,15 ---
  
    // Get cpool & tags index
    __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
    __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
    // See if bytecode has already been quicked
!   __ movzbl(rdx, Address(rdx, rbx,
!       Address::times_1,
!       Array<u1>::base_offset_in_bytes()));
!   __ andl (rdx, ~JVM_CONSTANT_QDescBit);
+   __ cmpl(rdx, JVM_CONSTANT_Class);
    __ jcc(Assembler::equal, quicked);
    __ push(atos); // save receiver for result, and for GC
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
  
    // vm_result_2 has metadata result

*** 4138,19 ***
    __ jump(ExternalAddress(Interpreter::_throw_ClassCastException_entry));
  
    // Come here on success
    __ bind(ok_is_subtype);
    __ mov(rax, rdx); // Restore object in rdx
  
    // Collect counts on whether this check-cast sees NULLs a lot or not.
    if (ProfileInterpreter) {
-     __ jmp(done);
-     __ bind(is_null);
      __ profile_null_seen(rcx);
-   } else {
-     __ bind(is_null);   // same as 'done'
    }
    __ bind(done);
  }
  
  void TemplateTable::instanceof() {
    transition(atos, itos);
--- 4454,33 ---
    __ jump(ExternalAddress(Interpreter::_throw_ClassCastException_entry));
  
    // Come here on success
    __ bind(ok_is_subtype);
    __ mov(rax, rdx); // Restore object in rdx
+   __ jmp(done);
+ 
+   __ bind(is_null);
  
    // Collect counts on whether this check-cast sees NULLs a lot or not.
    if (ProfileInterpreter) {
      __ profile_null_seen(rcx);
    }
+ 
+   if (EnableValhalla) {
+     // Get cpool & tags index
+     __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
+     __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
+     // See if CP entry is a Q-descriptor
+     __ movzbl(rcx, Address(rdx, rbx,
+         Address::times_1,
+         Array<u1>::base_offset_in_bytes()));
+     __ andl (rcx, JVM_CONSTANT_QDescBit);
+     __ cmpl(rcx, JVM_CONSTANT_QDescBit);
+     __ jcc(Assembler::notEqual, done);
+     __ jump(ExternalAddress(Interpreter::_throw_NullPointerException_entry));
+   }
+ 
    __ bind(done);
  }
  
  void TemplateTable::instanceof() {
    transition(atos, itos);

*** 4160,14 ***
  
    // Get cpool & tags index
    __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
    __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
    // See if bytecode has already been quicked
!   __ cmpb(Address(rdx, rbx,
!                   Address::times_1,
!                   Array<u1>::base_offset_in_bytes()),
!           JVM_CONSTANT_Class);
    __ jcc(Assembler::equal, quicked);
  
    __ push(atos); // save receiver for result, and for GC
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
    // vm_result_2 has metadata result
--- 4490,15 ---
  
    // Get cpool & tags index
    __ get_cpool_and_tags(rcx, rdx); // rcx=cpool, rdx=tags array
    __ get_unsigned_2_byte_index_at_bcp(rbx, 1); // rbx=index
    // See if bytecode has already been quicked
!   __ movzbl(rdx, Address(rdx, rbx,
!         Address::times_1,
!         Array<u1>::base_offset_in_bytes()));
!   __ andl (rdx, ~JVM_CONSTANT_QDescBit);
+   __ cmpl(rdx, JVM_CONSTANT_Class);
    __ jcc(Assembler::equal, quicked);
  
    __ push(atos); // save receiver for result, and for GC
    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
    // vm_result_2 has metadata result

*** 4216,11 ***
    __ bind(done);
    // rax = 0: obj == NULL or  obj is not an instanceof the specified klass
    // rax = 1: obj != NULL and obj is     an instanceof the specified klass
  }
  
- 
  //----------------------------------------------------------------------------------------------------
  // Breakpoints
  void TemplateTable::_breakpoint() {
    // Note: We get here even if we are single stepping..
    // jbug insists on setting breakpoints at every bytecode
--- 4547,10 ---

*** 4278,10 ***
--- 4608,14 ---
    transition(atos, vtos);
  
    // check for NULL object
    __ null_check(rax);
  
+   Label is_inline_type;
+   __ movptr(rbx, Address(rax, oopDesc::mark_offset_in_bytes()));
+   __ test_markword_is_inline_type(rbx, is_inline_type);
+ 
    const Address monitor_block_top(
          rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
    const Address monitor_block_bot(
          rbp, frame::interpreter_frame_initial_sp_offset * wordSize);
    const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;

*** 4367,18 ***
--- 4701,34 ---
    __ generate_stack_overflow_check(0);
  
    // The bcp has already been incremented. Just need to dispatch to
    // next instruction.
    __ dispatch_next(vtos);
+ 
+   __ bind(is_inline_type);
+   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
+                     InterpreterRuntime::throw_illegal_monitor_state_exception));
+   __ should_not_reach_here();
  }
  
  void TemplateTable::monitorexit() {
    transition(atos, vtos);
  
    // check for NULL object
    __ null_check(rax);
  
+   const int is_inline_type_mask = markWord::inline_type_pattern;
+   Label has_identity;
+   __ movptr(rbx, Address(rax, oopDesc::mark_offset_in_bytes()));
+   __ andptr(rbx, is_inline_type_mask);
+   __ cmpl(rbx, is_inline_type_mask);
+   __ jcc(Assembler::notEqual, has_identity);
+   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
+                      InterpreterRuntime::throw_illegal_monitor_state_exception));
+   __ should_not_reach_here();
+   __ bind(has_identity);
+ 
    const Address monitor_block_top(
          rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
    const Address monitor_block_bot(
          rbp, frame::interpreter_frame_initial_sp_offset * wordSize);
    const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
< prev index next >