< prev index next >

src/hotspot/share/runtime/sharedRuntime.cpp

Print this page
@@ -42,17 +42,22 @@
  #include "gc/shared/gcLocker.inline.hpp"
  #include "interpreter/interpreter.hpp"
  #include "interpreter/interpreterRuntime.hpp"
  #include "jfr/jfrEvents.hpp"
  #include "logging/log.hpp"
+ #include "memory/oopFactory.hpp"
  #include "memory/resourceArea.hpp"
  #include "memory/universe.hpp"
+ #include "oops/access.hpp"
+ #include "oops/fieldStreams.inline.hpp"
  #include "oops/compiledICHolder.inline.hpp"
  #include "oops/klass.hpp"
  #include "oops/method.inline.hpp"
  #include "oops/objArrayKlass.hpp"
+ #include "oops/objArrayOop.inline.hpp"
  #include "oops/oop.inline.hpp"
+ #include "oops/inlineKlass.inline.hpp"
  #include "prims/forte.hpp"
  #include "prims/jvmtiExport.hpp"
  #include "prims/methodHandles.hpp"
  #include "prims/nativeLookup.hpp"
  #include "runtime/atomic.hpp"

@@ -84,11 +89,10 @@
  RuntimeStub*        SharedRuntime::_wrong_method_abstract_blob;
  RuntimeStub*        SharedRuntime::_ic_miss_blob;
  RuntimeStub*        SharedRuntime::_resolve_opt_virtual_call_blob;
  RuntimeStub*        SharedRuntime::_resolve_virtual_call_blob;
  RuntimeStub*        SharedRuntime::_resolve_static_call_blob;
- address             SharedRuntime::_resolve_static_call_entry;
  
  DeoptimizationBlob* SharedRuntime::_deopt_blob;
  SafepointBlob*      SharedRuntime::_polling_page_vectors_safepoint_handler_blob;
  SafepointBlob*      SharedRuntime::_polling_page_safepoint_handler_blob;
  SafepointBlob*      SharedRuntime::_polling_page_return_handler_blob;

@@ -104,11 +108,10 @@
    _wrong_method_abstract_blob          = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_abstract), "wrong_method_abstract_stub");
    _ic_miss_blob                        = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_ic_miss),  "ic_miss_stub");
    _resolve_opt_virtual_call_blob       = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::resolve_opt_virtual_call_C),   "resolve_opt_virtual_call");
    _resolve_virtual_call_blob           = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::resolve_virtual_call_C),       "resolve_virtual_call");
    _resolve_static_call_blob            = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::resolve_static_call_C),        "resolve_static_call");
-   _resolve_static_call_entry           = _resolve_static_call_blob->entry_point();
  
    AdapterHandlerLibrary::initialize();
  
  #if COMPILER2_OR_JVMCI
    // Vectors are generated only by C2 and JVMCI.

@@ -984,10 +987,11 @@
    assert(oopDesc::is_oop(obj), "must be a valid oop");
    assert(obj->klass()->has_finalizer(), "shouldn't be here otherwise");
    InstanceKlass::register_finalizer(instanceOop(obj), CHECK);
  JRT_END
  
+ 
  jlong SharedRuntime::get_java_tid(Thread* thread) {
    if (thread != NULL) {
      if (thread->is_Java_thread()) {
        oop obj = JavaThread::cast(thread)->threadObj();
        return (obj == NULL) ? 0 : java_lang_Thread::thread_id(obj);

@@ -1084,10 +1088,25 @@
  
    // Find caller and bci from vframe
    methodHandle caller(current, vfst.method());
    int          bci   = vfst.bci();
  
+   // Substitutability test implementation piggy backs on static call resolution
+   Bytecodes::Code code = caller->java_code_at(bci);
+   if (code == Bytecodes::_if_acmpeq || code == Bytecodes::_if_acmpne) {
+     bc = Bytecodes::_invokestatic;
+     methodHandle attached_method(THREAD, extract_attached_method(vfst));
+     assert(attached_method.not_null(), "must have attached method");
+     vmClasses::PrimitiveObjectMethods_klass()->initialize(CHECK_NH);
+     LinkResolver::resolve_invoke(callinfo, receiver, attached_method, bc, false, CHECK_NH);
+ #ifdef ASSERT
+     Method* is_subst = vmClasses::PrimitiveObjectMethods_klass()->find_method(vmSymbols::isSubstitutable_name(), vmSymbols::object_object_boolean_signature());
+     assert(callinfo.selected_method() == is_subst, "must be isSubstitutable method");
+ #endif
+     return receiver;
+   }
+ 
    Bytecode_invoke bytecode(caller, bci);
    int bytecode_index = bytecode.index();
    bc = bytecode.invoke_code();
  
    methodHandle attached_method(current, extract_attached_method(vfst));

@@ -1119,56 +1138,78 @@
            }
            break;
          default:
            break;
        }
+     } else {
+       assert(attached_method->has_scalarized_args(), "invalid use of attached method");
+       if (!attached_method->method_holder()->is_inline_klass()) {
+         // Ignore the attached method in this case to not confuse below code
+         attached_method = methodHandle(current, NULL);
+       }
      }
    }
  
    assert(bc != Bytecodes::_illegal, "not initialized");
  
    bool has_receiver = bc != Bytecodes::_invokestatic &&
                        bc != Bytecodes::_invokedynamic &&
                        bc != Bytecodes::_invokehandle;
+   bool check_null_and_abstract = true;
  
    // Find receiver for non-static call
    if (has_receiver) {
      // This register map must be update since we need to find the receiver for
      // compiled frames. The receiver might be in a register.
      RegisterMap reg_map2(current);
      frame stubFrame   = current->last_frame();
      // Caller-frame is a compiled frame
      frame callerFrame = stubFrame.sender(&reg_map2);
+     bool caller_is_c1 = false;
  
-     if (attached_method.is_null()) {
-       Method* callee = bytecode.static_target(CHECK_NH);
+     if (callerFrame.is_compiled_frame() && !callerFrame.is_deoptimized_frame()) {
+       caller_is_c1 = callerFrame.cb()->is_compiled_by_c1();
+     }
+ 
+     Method* callee = attached_method();
+     if (callee == NULL) {
+       callee = bytecode.static_target(CHECK_NH);
        if (callee == NULL) {
          THROW_(vmSymbols::java_lang_NoSuchMethodException(), nullHandle);
        }
      }
- 
-     // Retrieve from a compiled argument list
-     receiver = Handle(current, callerFrame.retrieve_receiver(&reg_map2));
- 
-     if (receiver.is_null()) {
-       THROW_(vmSymbols::java_lang_NullPointerException(), nullHandle);
+     if (!caller_is_c1 && callee->has_scalarized_args() && callee->method_holder()->is_inline_klass() &&
+         InlineKlass::cast(callee->method_holder())->can_be_passed_as_fields()) {
+       // If the receiver is an inline type that is passed as fields, no oop is available
+       // Resolve the call without receiver null checking.
+       assert(attached_method.not_null() && !attached_method->is_abstract(), "must have non-abstract attached method");
+       if (bc == Bytecodes::_invokeinterface) {
+         bc = Bytecodes::_invokevirtual; // C2 optimistically replaces interface calls by virtual calls
+       }
+       check_null_and_abstract = false;
+     } else {
+       // Retrieve from a compiled argument list
+       receiver = Handle(current, callerFrame.retrieve_receiver(&reg_map2));
+       if (receiver.is_null()) {
+         THROW_(vmSymbols::java_lang_NullPointerException(), nullHandle);
+       }
      }
    }
  
    // Resolve method
    if (attached_method.not_null()) {
      // Parameterized by attached method.
-     LinkResolver::resolve_invoke(callinfo, receiver, attached_method, bc, CHECK_NH);
+     LinkResolver::resolve_invoke(callinfo, receiver, attached_method, bc, check_null_and_abstract, CHECK_NH);
    } else {
      // Parameterized by bytecode.
      constantPoolHandle constants(current, caller->constants());
      LinkResolver::resolve_invoke(callinfo, receiver, constants, bytecode_index, bc, CHECK_NH);
    }
  
  #ifdef ASSERT
    // Check that the receiver klass is of the right subtype and that it is initialized for virtual calls
-   if (has_receiver) {
+   if (has_receiver && check_null_and_abstract) {
      assert(receiver.not_null(), "should have thrown exception");
      Klass* receiver_klass = receiver->klass();
      Klass* rk = NULL;
      if (attached_method.not_null()) {
        // In case there's resolved method attached, use its holder during the check.

@@ -1222,13 +1263,13 @@
    assert(callee_method()->is_method(), "must be");
    return callee_method;
  }
  
  // Resolves a call.
- methodHandle SharedRuntime::resolve_helper(bool is_virtual, bool is_optimized, TRAPS) {
+ methodHandle SharedRuntime::resolve_helper(bool is_virtual, bool is_optimized, bool* caller_is_c1, TRAPS) {
    methodHandle callee_method;
-   callee_method = resolve_sub_helper(is_virtual, is_optimized, THREAD);
+   callee_method = resolve_sub_helper(is_virtual, is_optimized, caller_is_c1, THREAD);
    if (JvmtiExport::can_hotswap_or_post_breakpoint()) {
      int retry_count = 0;
      while (!HAS_PENDING_EXCEPTION && callee_method->is_old() &&
             callee_method->method_holder() != vmClasses::Object_klass()) {
        // If has a pending exception then there is no need to re-try to

@@ -1241,11 +1282,11 @@
        // in the middle of resolve. If it is looping here more than 100 times
        // means then there could be a bug here.
        guarantee((retry_count++ < 100),
                  "Could not resolve to latest version of redefined method");
        // method is redefined in the middle of resolve so re-try.
-       callee_method = resolve_sub_helper(is_virtual, is_optimized, THREAD);
+       callee_method = resolve_sub_helper(is_virtual, is_optimized, caller_is_c1, THREAD);
      }
    }
    return callee_method;
  }
  

@@ -1272,21 +1313,29 @@
  #ifdef ASSERT
    address dest_entry_point = callee == NULL ? 0 : callee->entry_point(); // used below
  #endif
  
    bool is_nmethod = caller_nm->is_nmethod();
+   bool caller_is_c1 = caller_nm->is_compiled_by_c1();
  
    if (is_virtual) {
-     assert(receiver.not_null() || invoke_code == Bytecodes::_invokehandle, "sanity check");
+     Klass* receiver_klass = NULL;
+     if (!caller_is_c1 && callee_method->has_scalarized_args() && callee_method->method_holder()->is_inline_klass() &&
+         InlineKlass::cast(callee_method->method_holder())->can_be_passed_as_fields()) {
+       // If the receiver is an inline type that is passed as fields, no oop is available
+       receiver_klass = callee_method->method_holder();
+     } else {
+       assert(receiver.not_null() || invoke_code == Bytecodes::_invokehandle, "sanity check");
+       receiver_klass = invoke_code == Bytecodes::_invokehandle ? NULL : receiver->klass();
+     }
      bool static_bound = call_info.resolved_method()->can_be_statically_bound();
-     Klass* klass = invoke_code == Bytecodes::_invokehandle ? NULL : receiver->klass();
-     CompiledIC::compute_monomorphic_entry(callee_method, klass,
-                      is_optimized, static_bound, is_nmethod, virtual_call_info,
+     CompiledIC::compute_monomorphic_entry(callee_method, receiver_klass,
+                      is_optimized, static_bound, is_nmethod, caller_is_c1, virtual_call_info,
                       CHECK_false);
    } else {
      // static call
-     CompiledStaticCall::compute_entry(callee_method, is_nmethod, static_call_info);
+     CompiledStaticCall::compute_entry(callee_method, caller_nm, static_call_info);
    }
  
    // grab lock, check for deoptimization and potentially patch caller
    {
      CompiledICLocker ml(caller_nm);

@@ -1332,19 +1381,20 @@
    return true;
  }
  
  // Resolves a call.  The compilers generate code for calls that go here
  // and are patched with the real destination of the call.
- methodHandle SharedRuntime::resolve_sub_helper(bool is_virtual, bool is_optimized, TRAPS) {
+ methodHandle SharedRuntime::resolve_sub_helper(bool is_virtual, bool is_optimized, bool* caller_is_c1, TRAPS) {
    JavaThread* current = THREAD;
    ResourceMark rm(current);
    RegisterMap cbl_map(current, false);
    frame caller_frame = current->last_frame().sender(&cbl_map);
  
    CodeBlob* caller_cb = caller_frame.cb();
    guarantee(caller_cb != NULL && caller_cb->is_compiled(), "must be called from compiled method");
    CompiledMethod* caller_nm = caller_cb->as_compiled_method_or_null();
+   *caller_is_c1 = caller_nm->is_compiled_by_c1();
  
    // make sure caller is not getting deoptimized
    // and removed before we are done with it.
    // CLEANUP - with lazy deopt shouldn't need this lock
    nmethodLocker caller_lock(caller_nm);

@@ -1437,22 +1487,23 @@
  #ifdef ASSERT
    RegisterMap reg_map(current, false);
    frame stub_frame = current->last_frame();
    assert(stub_frame.is_runtime_frame(), "sanity check");
    frame caller_frame = stub_frame.sender(&reg_map);
-   assert(!caller_frame.is_interpreted_frame() && !caller_frame.is_entry_frame() && !caller_frame.is_optimized_entry_frame(), "unexpected frame");
+   assert(!caller_frame.is_interpreted_frame() && !caller_frame.is_entry_frame()  && !caller_frame.is_optimized_entry_frame(), "unexpected frame");
  #endif /* ASSERT */
  
    methodHandle callee_method;
+   bool is_optimized = false;
+   bool caller_is_c1 = false;
    JRT_BLOCK
-     callee_method = SharedRuntime::handle_ic_miss_helper(CHECK_NULL);
+     callee_method = SharedRuntime::handle_ic_miss_helper(is_optimized, caller_is_c1, CHECK_NULL);
      // Return Method* through TLS
      current->set_vm_result_2(callee_method());
    JRT_BLOCK_END
    // return compiled code entry point after potential safepoints
-   assert(callee_method->verified_code_entry() != NULL, " Jump to zero!");
-   return callee_method->verified_code_entry();
+   return entry_for_handle_wrong_method(callee_method, false, is_optimized, caller_is_c1);
  JRT_END
  
  
  // Handle call site that has been made non-entrant
  JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method(JavaThread* current))

@@ -1492,18 +1543,20 @@
      }
    }
  
    // Must be compiled to compiled path which is safe to stackwalk
    methodHandle callee_method;
+   bool is_static_call = false;
+   bool is_optimized = false;
+   bool caller_is_c1 = false;
    JRT_BLOCK
      // Force resolving of caller (if we called from compiled frame)
-     callee_method = SharedRuntime::reresolve_call_site(CHECK_NULL);
+     callee_method = SharedRuntime::reresolve_call_site(is_static_call, is_optimized, caller_is_c1, CHECK_NULL);
      current->set_vm_result_2(callee_method());
    JRT_BLOCK_END
    // return compiled code entry point after potential safepoints
-   assert(callee_method->verified_code_entry() != NULL, " Jump to zero!");
-   return callee_method->verified_code_entry();
+   return entry_for_handle_wrong_method(callee_method, is_static_call, is_optimized, caller_is_c1);
  JRT_END
  
  // Handle abstract method call
  JRT_BLOCK_ENTRY(address, SharedRuntime::handle_wrong_method_abstract(JavaThread* current))
    // Verbose error message for AbstractMethodError.

@@ -1537,65 +1590,75 @@
  
  
  // resolve a static call and patch code
  JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_static_call_C(JavaThread* current ))
    methodHandle callee_method;
+   bool caller_is_c1;
    JRT_BLOCK
-     callee_method = SharedRuntime::resolve_helper(false, false, CHECK_NULL);
+     callee_method = SharedRuntime::resolve_helper(false, false, &caller_is_c1, CHECK_NULL);
      current->set_vm_result_2(callee_method());
    JRT_BLOCK_END
    // return compiled code entry point after potential safepoints
-   assert(callee_method->verified_code_entry() != NULL, " Jump to zero!");
-   return callee_method->verified_code_entry();
+   address entry = caller_is_c1 ?
+     callee_method->verified_inline_code_entry() : callee_method->verified_code_entry();
+   assert(entry != NULL, "Jump to zero!");
+   return entry;
  JRT_END
  
  
  // resolve virtual call and update inline cache to monomorphic
  JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_virtual_call_C(JavaThread* current))
    methodHandle callee_method;
+   bool caller_is_c1;
    JRT_BLOCK
-     callee_method = SharedRuntime::resolve_helper(true, false, CHECK_NULL);
+     callee_method = SharedRuntime::resolve_helper(true, false, &caller_is_c1, CHECK_NULL);
      current->set_vm_result_2(callee_method());
    JRT_BLOCK_END
    // return compiled code entry point after potential safepoints
-   assert(callee_method->verified_code_entry() != NULL, " Jump to zero!");
-   return callee_method->verified_code_entry();
+   address entry = caller_is_c1 ?
+     callee_method->verified_inline_code_entry() : callee_method->verified_inline_ro_code_entry();
+   assert(entry != NULL, "Jump to zero!");
+   return entry;
  JRT_END
  
  
  // Resolve a virtual call that can be statically bound (e.g., always
  // monomorphic, so it has no inline cache).  Patch code to resolved target.
  JRT_BLOCK_ENTRY(address, SharedRuntime::resolve_opt_virtual_call_C(JavaThread* current))
    methodHandle callee_method;
+   bool caller_is_c1;
    JRT_BLOCK
-     callee_method = SharedRuntime::resolve_helper(true, true, CHECK_NULL);
+     callee_method = SharedRuntime::resolve_helper(true, true, &caller_is_c1, CHECK_NULL);
      current->set_vm_result_2(callee_method());
    JRT_BLOCK_END
    // return compiled code entry point after potential safepoints
-   assert(callee_method->verified_code_entry() != NULL, " Jump to zero!");
-   return callee_method->verified_code_entry();
+   address entry = caller_is_c1 ?
+     callee_method->verified_inline_code_entry() : callee_method->verified_code_entry();
+   assert(entry != NULL, "Jump to zero!");
+   return entry;
  JRT_END
  
  // The handle_ic_miss_helper_internal function returns false if it failed due
  // to either running out of vtable stubs or ic stubs due to IC transitions
  // to transitional states. The needs_ic_stub_refill value will be set if
  // the failure was due to running out of IC stubs, in which case handle_ic_miss_helper
  // refills the IC stubs and tries again.
  bool SharedRuntime::handle_ic_miss_helper_internal(Handle receiver, CompiledMethod* caller_nm,
                                                     const frame& caller_frame, methodHandle callee_method,
                                                     Bytecodes::Code bc, CallInfo& call_info,
-                                                    bool& needs_ic_stub_refill, TRAPS) {
+                                                    bool& needs_ic_stub_refill, bool& is_optimized, bool caller_is_c1, TRAPS) {
    CompiledICLocker ml(caller_nm);
    CompiledIC* inline_cache = CompiledIC_before(caller_nm, caller_frame.pc());
    bool should_be_mono = false;
    if (inline_cache->is_optimized()) {
      if (TraceCallFixup) {
        ResourceMark rm(THREAD);
        tty->print("OPTIMIZED IC miss (%s) call to", Bytecodes::name(bc));
        callee_method->print_short_name(tty);
        tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code()));
      }
+     is_optimized = true;
      should_be_mono = true;
    } else if (inline_cache->is_icholder_call()) {
      CompiledICHolder* ic_oop = inline_cache->cached_icholder();
      if (ic_oop != NULL) {
        if (!ic_oop->is_loader_alive()) {

@@ -1629,19 +1692,20 @@
      Klass* receiver_klass = receiver()->klass();
      inline_cache->compute_monomorphic_entry(callee_method,
                                              receiver_klass,
                                              inline_cache->is_optimized(),
                                              false, caller_nm->is_nmethod(),
+                                             caller_nm->is_compiled_by_c1(),
                                              info, CHECK_false);
      if (!inline_cache->set_to_monomorphic(info)) {
        needs_ic_stub_refill = true;
        return false;
      }
    } else if (!inline_cache->is_megamorphic() && !inline_cache->is_clean()) {
      // Potential change to megamorphic
  
-     bool successful = inline_cache->set_to_megamorphic(&call_info, bc, needs_ic_stub_refill, CHECK_false);
+     bool successful = inline_cache->set_to_megamorphic(&call_info, bc, needs_ic_stub_refill, caller_is_c1, CHECK_false);
      if (needs_ic_stub_refill) {
        return false;
      }
      if (!successful) {
        if (!inline_cache->set_to_clean()) {

@@ -1653,11 +1717,11 @@
      // Either clean or megamorphic
    }
    return true;
  }
  
- methodHandle SharedRuntime::handle_ic_miss_helper(TRAPS) {
+ methodHandle SharedRuntime::handle_ic_miss_helper(bool& is_optimized, bool& caller_is_c1, TRAPS) {
    JavaThread* current = THREAD;
    ResourceMark rm(current);
    CallInfo call_info;
    Bytecodes::Code bc;
  

@@ -1673,11 +1737,13 @@
    // plain ic_miss) and the site will be converted to an optimized virtual call site
    // never to miss again. I don't believe C2 will produce code like this but if it
    // did this would still be the correct thing to do for it too, hence no ifdef.
    //
    if (call_info.resolved_method()->can_be_statically_bound()) {
-     methodHandle callee_method = SharedRuntime::reresolve_call_site(CHECK_(methodHandle()));
+     bool is_static_call = false;
+     methodHandle callee_method = SharedRuntime::reresolve_call_site(is_static_call, is_optimized, caller_is_c1, CHECK_(methodHandle()));
+     assert(!is_static_call, "IC miss at static call?");
      if (TraceCallFixup) {
        RegisterMap reg_map(current, false);
        frame caller_frame = current->last_frame().sender(&reg_map);
        ResourceMark rm(current);
        tty->print("converting IC miss to reresolve (%s) call to", Bytecodes::name(bc));

@@ -1723,16 +1789,17 @@
    // that refills them.
    RegisterMap reg_map(current, false);
    frame caller_frame = current->last_frame().sender(&reg_map);
    CodeBlob* cb = caller_frame.cb();
    CompiledMethod* caller_nm = cb->as_compiled_method();
+   caller_is_c1 = caller_nm->is_compiled_by_c1();
  
    for (;;) {
      ICRefillVerifier ic_refill_verifier;
      bool needs_ic_stub_refill = false;
      bool successful = handle_ic_miss_helper_internal(receiver, caller_nm, caller_frame, callee_method,
-                                                      bc, call_info, needs_ic_stub_refill, CHECK_(methodHandle()));
+                                                      bc, call_info, needs_ic_stub_refill, is_optimized, caller_is_c1, CHECK_(methodHandle()));
      if (successful || !needs_ic_stub_refill) {
        return callee_method;
      } else {
        InlineCacheBuffer::refill_ic_stubs();
      }

@@ -1760,11 +1827,11 @@
  // Resets a call-site in compiled code so it will get resolved again.
  // This routines handles both virtual call sites, optimized virtual call
  // sites, and static call sites. Typically used to change a call sites
  // destination from compiled to interpreted.
  //
- methodHandle SharedRuntime::reresolve_call_site(TRAPS) {
+ methodHandle SharedRuntime::reresolve_call_site(bool& is_static_call, bool& is_optimized, bool& caller_is_c1, TRAPS) {
    JavaThread* current = THREAD;
    ResourceMark rm(current);
    RegisterMap reg_map(current, false);
    frame stub_frame = current->last_frame();
    assert(stub_frame.is_runtime_frame(), "must be a runtimeStub");

@@ -1777,12 +1844,12 @@
    if (caller.is_compiled_frame() && !caller.is_deoptimized_frame()) {
  
      address pc = caller.pc();
  
      // Check for static or virtual call
-     bool is_static_call = false;
      CompiledMethod* caller_nm = CodeCache::find_compiled(pc);
+     caller_is_c1 = caller_nm->is_compiled_by_c1();
  
      // Default call_addr is the location of the "basic" call.
      // Determine the address of the call we a reresolving. With
      // Inline Caches we will always find a recognizable call.
      // With Inline Caches disabled we may or may not find a

@@ -1822,10 +1889,11 @@
            is_static_call = true;
          } else {
            assert(iter.type() == relocInfo::virtual_call_type ||
                   iter.type() == relocInfo::opt_virtual_call_type
                  , "unexpected relocInfo. type");
+           is_optimized = (iter.type() == relocInfo::opt_virtual_call_type);
          }
        } else {
          assert(!UseInlineCaches, "relocation info. must exist for this address");
        }
  

@@ -1847,11 +1915,10 @@
      }
    }
  
    methodHandle callee_method = find_callee_method(CHECK_(methodHandle()));
  
- 
  #ifndef PRODUCT
    Atomic::inc(&_wrong_method_ctr);
  
    if (TraceCallFixup) {
      ResourceMark rm(current);

@@ -1941,12 +2008,10 @@
  // interpreted. If the caller is compiled we attempt to patch the caller
  // so he no longer calls into the interpreter.
  JRT_LEAF(void, SharedRuntime::fixup_callers_callsite(Method* method, address caller_pc))
    Method* moop(method);
  
-   address entry_point = moop->from_compiled_entry_no_trampoline();
- 
    // It's possible that deoptimization can occur at a call site which hasn't
    // been resolved yet, in which case this function will be called from
    // an nmethod that has been patched for deopt and we can ignore the
    // request for a fixup.
    // Also it is possible that we lost a race in that from_compiled_entry

@@ -1954,11 +2019,15 @@
    // we did we'd leap into space because the callsite needs to use
    // "to interpreter" stub in order to load up the Method*. Don't
    // ask me how I know this...
  
    CodeBlob* cb = CodeCache::find_blob(caller_pc);
-   if (cb == NULL || !cb->is_compiled() || entry_point == moop->get_c2i_entry()) {
+   if (cb == NULL || !cb->is_compiled()) {
+     return;
+   }
+   address entry_point = moop->from_compiled_entry_no_trampoline(cb->is_compiled_by_c1());
+   if (entry_point == moop->get_c2i_entry()) {
      return;
    }
  
    // The check above makes sure this is a nmethod.
    CompiledMethod* nm = cb->as_compiled_method_or_null();

@@ -2329,17 +2398,17 @@
                 // Otherwise _value._fingerprint is the array.
  
    // Remap BasicTypes that are handled equivalently by the adapters.
    // These are correct for the current system but someday it might be
    // necessary to make this mapping platform dependent.
-   static int adapter_encoding(BasicType in) {
+   static BasicType adapter_encoding(BasicType in) {
      switch (in) {
        case T_BOOLEAN:
        case T_BYTE:
        case T_SHORT:
        case T_CHAR:
-         // There are all promoted to T_INT in the calling convention
+         // They are all promoted to T_INT in the calling convention
          return T_INT;
  
        case T_OBJECT:
        case T_ARRAY:
          // In other words, we assume that any register good enough for

@@ -2362,13 +2431,14 @@
          return T_CONFLICT;
      }
    }
  
   public:
-   AdapterFingerPrint(int total_args_passed, BasicType* sig_bt) {
+   AdapterFingerPrint(const GrowableArray<SigEntry>* sig, bool has_ro_adapter = false) {
      // The fingerprint is based on the BasicType signature encoded
      // into an array of ints with eight entries per int.
+     int total_args_passed = (sig != NULL) ? sig->length() : 0;
      int* ptr;
      int len = (total_args_passed + (_basic_types_per_int-1)) / _basic_types_per_int;
      if (len <= _compact_int_count) {
        assert(_compact_int_count == 3, "else change next line");
        _value._compact[0] = _value._compact[1] = _value._compact[2] = 0;

@@ -2382,19 +2452,45 @@
        ptr = _value._fingerprint;
      }
  
      // Now pack the BasicTypes with 8 per int
      int sig_index = 0;
+     BasicType prev_bt = T_ILLEGAL;
+     int vt_count = 0;
      for (int index = 0; index < len; index++) {
        int value = 0;
-       for (int byte = 0; sig_index < total_args_passed && byte < _basic_types_per_int; byte++) {
-         int bt = adapter_encoding(sig_bt[sig_index++]);
-         assert((bt & _basic_type_mask) == bt, "must fit in 4 bits");
-         value = (value << _basic_type_bits) | bt;
+       for (int byte = 0; byte < _basic_types_per_int; byte++) {
+         BasicType bt = T_ILLEGAL;
+         if (sig_index < total_args_passed) {
+           bt = sig->at(sig_index++)._bt;
+           if (bt == T_INLINE_TYPE) {
+             // Found start of inline type in signature
+             assert(InlineTypePassFieldsAsArgs, "unexpected start of inline type");
+             if (sig_index == 1 && has_ro_adapter) {
+               // With a ro_adapter, replace receiver inline type delimiter by T_VOID to prevent matching
+               // with other adapters that have the same inline type as first argument and no receiver.
+               bt = T_VOID;
+             }
+             vt_count++;
+           } else if (bt == T_VOID && prev_bt != T_LONG && prev_bt != T_DOUBLE) {
+             // Found end of inline type in signature
+             assert(InlineTypePassFieldsAsArgs, "unexpected end of inline type");
+             vt_count--;
+             assert(vt_count >= 0, "invalid vt_count");
+           } else if (vt_count == 0) {
+             // Widen fields that are not part of a scalarized inline type argument
+             bt = adapter_encoding(bt);
+           }
+           prev_bt = bt;
+         }
+         int bt_val = (bt == T_ILLEGAL) ? 0 : bt;
+         assert((bt_val & _basic_type_mask) == bt_val, "must fit in 4 bits");
+         value = (value << _basic_type_bits) | bt_val;
        }
        ptr[index] = value;
      }
+     assert(vt_count == 0, "invalid vt_count");
    }
  
    ~AdapterFingerPrint() {
      if (_length > 0) {
        FREE_C_HEAP_ARRAY(int, _value._fingerprint);

@@ -2517,13 +2613,16 @@
   public:
    AdapterHandlerTable()
      : BasicHashtable<mtCode>(293, (sizeof(AdapterHandlerEntry))) { }
  
    // Create a new entry suitable for insertion in the table
-   AdapterHandlerEntry* new_entry(AdapterFingerPrint* fingerprint, address i2c_entry, address c2i_entry, address c2i_unverified_entry, address c2i_no_clinit_check_entry) {
+   AdapterHandlerEntry* new_entry(AdapterFingerPrint* fingerprint, address i2c_entry, address c2i_entry,
+                                  address c2i_inline_entry, address c2i_inline_ro_entry,
+                                  address c2i_unverified_entry, address c2i_unverified_inline_entry, address c2i_no_clinit_check_entry) {
      AdapterHandlerEntry* entry = (AdapterHandlerEntry*)BasicHashtable<mtCode>::new_entry(fingerprint->compute_hash());
-     entry->init(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
+     entry->init(fingerprint, i2c_entry, c2i_entry, c2i_inline_entry, c2i_inline_ro_entry,
+                 c2i_unverified_entry, c2i_unverified_inline_entry, c2i_no_clinit_check_entry);
      return entry;
    }
  
    // Insert an entry into the table
    void add(AdapterHandlerEntry* entry) {

@@ -2535,13 +2634,13 @@
      entry->deallocate();
      BasicHashtable<mtCode>::free_entry(entry);
    }
  
    // Find a entry with the same fingerprint if it exists
-   AdapterHandlerEntry* lookup(int total_args_passed, BasicType* sig_bt) {
+   AdapterHandlerEntry* lookup(const GrowableArray<SigEntry>* sig, bool has_ro_adapter = false) {
      NOT_PRODUCT(_lookups++);
-     AdapterFingerPrint fp(total_args_passed, sig_bt);
+     AdapterFingerPrint fp(sig, has_ro_adapter);
      unsigned int hash = fp.compute_hash();
      int index = hash_to_index(hash);
      for (AdapterHandlerEntry* e = bucket(index); e != NULL; e = e->next()) {
        NOT_PRODUCT(_buckets++);
        if (e->hash() == hash) {

@@ -2638,11 +2737,11 @@
  AdapterHandlerEntry* AdapterHandlerLibrary::_no_arg_handler = NULL;
  AdapterHandlerEntry* AdapterHandlerLibrary::_int_arg_handler = NULL;
  AdapterHandlerEntry* AdapterHandlerLibrary::_obj_arg_handler = NULL;
  AdapterHandlerEntry* AdapterHandlerLibrary::_obj_int_arg_handler = NULL;
  AdapterHandlerEntry* AdapterHandlerLibrary::_obj_obj_arg_handler = NULL;
- const int AdapterHandlerLibrary_size = 16*K;
+ const int AdapterHandlerLibrary_size = 32*K;
  BufferBlob* AdapterHandlerLibrary::_buffer = NULL;
  
  BufferBlob* AdapterHandlerLibrary::buffer_blob() {
    return _buffer;
  }

@@ -2682,36 +2781,49 @@
      // are never compiled so an i2c entry is somewhat meaningless, but
      // throw AbstractMethodError just in case.
      // Pass wrong_method_abstract for the c2i transitions to return
      // AbstractMethodError for invalid invocations.
      address wrong_method_abstract = SharedRuntime::get_handle_wrong_method_abstract_stub();
-     _abstract_method_handler = AdapterHandlerLibrary::new_entry(new AdapterFingerPrint(0, NULL),
+     _abstract_method_handler = AdapterHandlerLibrary::new_entry(new AdapterFingerPrint(NULL),
                                                                  StubRoutines::throw_AbstractMethodError_entry(),
+                                                                 wrong_method_abstract, wrong_method_abstract, wrong_method_abstract,
                                                                  wrong_method_abstract, wrong_method_abstract);
- 
      _buffer = BufferBlob::create("adapters", AdapterHandlerLibrary_size);
  
-     _no_arg_handler = create_adapter(no_arg_blob, 0, NULL, true);
+     CompiledEntrySignature no_args;
+     no_args.compute_calling_conventions();
+     _no_arg_handler = create_adapter(no_arg_blob, no_args, true);
  
-     BasicType obj_args[] = { T_OBJECT };
-     _obj_arg_handler = create_adapter(obj_arg_blob, 1, obj_args, true);
+     CompiledEntrySignature obj_args;
+     SigEntry::add_entry(&obj_args.sig(), T_OBJECT, NULL);
+     obj_args.compute_calling_conventions();
+     _obj_arg_handler = create_adapter(obj_arg_blob, obj_args, true);
  
-     BasicType int_args[] = { T_INT };
-     _int_arg_handler = create_adapter(int_arg_blob, 1, int_args, true);
+     CompiledEntrySignature int_args;
+     SigEntry::add_entry(&int_args.sig(), T_INT, NULL);
+     int_args.compute_calling_conventions();
+     _int_arg_handler = create_adapter(int_arg_blob, int_args, true);
  
-     BasicType obj_int_args[] = { T_OBJECT, T_INT };
-     _obj_int_arg_handler = create_adapter(obj_int_arg_blob, 2, obj_int_args, true);
+     CompiledEntrySignature obj_int_args;
+     SigEntry::add_entry(&obj_int_args.sig(), T_OBJECT, NULL);
+     SigEntry::add_entry(&obj_int_args.sig(), T_INT, NULL);
+     obj_int_args.compute_calling_conventions();
+     _obj_int_arg_handler = create_adapter(obj_int_arg_blob, obj_int_args, true);
  
-     BasicType obj_obj_args[] = { T_OBJECT, T_OBJECT };
-     _obj_obj_arg_handler = create_adapter(obj_obj_arg_blob, 2, obj_obj_args, true);
+     CompiledEntrySignature obj_obj_args;
+     SigEntry::add_entry(&obj_obj_args.sig(), T_OBJECT, NULL);
+     SigEntry::add_entry(&obj_obj_args.sig(), T_OBJECT, NULL);
+     obj_obj_args.compute_calling_conventions();
+     _obj_obj_arg_handler = create_adapter(obj_obj_arg_blob, obj_obj_args, true);
  
      assert(no_arg_blob != NULL &&
            obj_arg_blob != NULL &&
            int_arg_blob != NULL &&
            obj_int_arg_blob != NULL &&
            obj_obj_arg_blob != NULL, "Initial adapters must be properly created");
    }
+   return;
  
    // Outside of the lock
    post_adapter_creation(no_arg_blob, _no_arg_handler);
    post_adapter_creation(obj_arg_blob, _obj_arg_handler);
    post_adapter_creation(int_arg_blob, _int_arg_handler);

@@ -2720,24 +2832,28 @@
  }
  
  AdapterHandlerEntry* AdapterHandlerLibrary::new_entry(AdapterFingerPrint* fingerprint,
                                                        address i2c_entry,
                                                        address c2i_entry,
+                                                       address c2i_inline_entry,
+                                                       address c2i_inline_ro_entry,
                                                        address c2i_unverified_entry,
+                                                       address c2i_unverified_inline_entry,
                                                        address c2i_no_clinit_check_entry) {
-   return _adapters->new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
+   return _adapters->new_entry(fingerprint, i2c_entry, c2i_entry, c2i_inline_entry, c2i_inline_ro_entry, c2i_unverified_entry,
+                               c2i_unverified_inline_entry, c2i_no_clinit_check_entry);
  }
  
  AdapterHandlerEntry* AdapterHandlerLibrary::get_simple_adapter(const methodHandle& method) {
    if (method->is_abstract()) {
-     return _abstract_method_handler;
+     return NULL;
    }
    int total_args_passed = method->size_of_parameters(); // All args on stack
    if (total_args_passed == 0) {
      return _no_arg_handler;
    } else if (total_args_passed == 1) {
-     if (!method->is_static()) {
+     if (!method->is_static() && !method->method_holder()->is_inline_klass()) {
        return _obj_arg_handler;
      }
      switch (method->signature()->char_at(1)) {
        case JVM_SIGNATURE_CLASS:
        case JVM_SIGNATURE_ARRAY:

@@ -2748,11 +2864,11 @@
        case JVM_SIGNATURE_BYTE:
        case JVM_SIGNATURE_SHORT:
          return _int_arg_handler;
      }
    } else if (total_args_passed == 2 &&
-              !method->is_static()) {
+              !method->is_static() && !method->method_holder()->is_inline_klass()) {
      switch (method->signature()->char_at(1)) {
        case JVM_SIGNATURE_CLASS:
        case JVM_SIGNATURE_ARRAY:
          return _obj_obj_arg_handler;
        case JVM_SIGNATURE_INT:

@@ -2764,51 +2880,148 @@
      }
    }
    return NULL;
  }
  
- class AdapterSignatureIterator : public SignatureIterator {
-  private:
-   BasicType stack_sig_bt[16];
-   BasicType* sig_bt;
-   int index;
+ CompiledEntrySignature::CompiledEntrySignature(Method* method) :
+   _method(method), _num_inline_args(0), _has_inline_recv(false),
+   _regs(NULL), _regs_cc(NULL), _regs_cc_ro(NULL),
+   _args_on_stack(0), _args_on_stack_cc(0), _args_on_stack_cc_ro(0),
+   _c1_needs_stack_repair(false), _c2_needs_stack_repair(false) {
+   _sig = new GrowableArray<SigEntry>((method != NULL) ? method->size_of_parameters() : 1);
+   _sig_cc = _sig;
+   _sig_cc_ro = _sig;
+ }
  
-  public:
-   AdapterSignatureIterator(Symbol* signature,
-                            fingerprint_t fingerprint,
-                            bool is_static,
-                            int total_args_passed) :
-     SignatureIterator(signature, fingerprint),
-     index(0)
-   {
-     sig_bt = (total_args_passed <= 16) ? stack_sig_bt : NEW_RESOURCE_ARRAY(BasicType, total_args_passed);
-     if (!is_static) { // Pass in receiver first
-       sig_bt[index++] = T_OBJECT;
+ int CompiledEntrySignature::compute_scalarized_cc(GrowableArray<SigEntry>*& sig_cc, VMRegPair*& regs_cc, bool scalar_receiver) {
+   InstanceKlass* holder = _method->method_holder();
+   sig_cc = new GrowableArray<SigEntry>(_method->size_of_parameters());
+   if (!_method->is_static()) {
+     if (holder->is_inline_klass() && scalar_receiver && InlineKlass::cast(holder)->can_be_passed_as_fields()) {
+       sig_cc->appendAll(InlineKlass::cast(holder)->extended_sig());
+     } else {
+       SigEntry::add_entry(sig_cc, T_OBJECT, holder->name());
+     }
+   }
+   for (SignatureStream ss(_method->signature()); !ss.at_return_type(); ss.next()) {
+     if (ss.type() == T_INLINE_TYPE) {
+       InlineKlass* vk = ss.as_inline_klass(holder);
+       if (vk->can_be_passed_as_fields()) {
+         sig_cc->appendAll(vk->extended_sig());
+       } else {
+         SigEntry::add_entry(sig_cc, T_OBJECT, ss.as_symbol());
+       }
+     } else {
+       SigEntry::add_entry(sig_cc, ss.type(), ss.as_symbol());
      }
-     do_parameters_on(this);
    }
+   regs_cc = NEW_RESOURCE_ARRAY(VMRegPair, sig_cc->length() + 2);
+   return SharedRuntime::java_calling_convention(sig_cc, regs_cc);
+ }
  
-   BasicType* basic_types() {
-     return sig_bt;
+ // See if we can save space by sharing the same entry for VIEP and VIEP(RO),
+ // or the same entry for VEP and VIEP(RO).
+ CodeOffsets::Entries CompiledEntrySignature::c1_inline_ro_entry_type() const {
+   if (!has_scalarized_args()) {
+     // VEP/VIEP/VIEP(RO) all share the same entry. There's no packing.
+     return CodeOffsets::Verified_Entry;
+   }
+   if (_method->is_static()) {
+     // Static methods don't need VIEP(RO)
+     return CodeOffsets::Verified_Entry;
    }
  
- #ifdef ASSERT
-   int slots() {
-     return index;
+   if (has_inline_recv()) {
+     if (num_inline_args() == 1) {
+       // Share same entry for VIEP and VIEP(RO).
+       // This is quite common: we have an instance method in an InlineKlass that has
+       // no inline type args other than <this>.
+       return CodeOffsets::Verified_Inline_Entry;
+     } else {
+       assert(num_inline_args() > 1, "must be");
+       // No sharing:
+       //   VIEP(RO) -- <this> is passed as object
+       //   VEP      -- <this> is passed as fields
+       return CodeOffsets::Verified_Inline_Entry_RO;
+     }
    }
- #endif
  
-  private:
+   // Either a static method, or <this> is not an inline type
+   if (args_on_stack_cc() != args_on_stack_cc_ro()) {
+     // No sharing:
+     // Some arguments are passed on the stack, and we have inserted reserved entries
+     // into the VEP, but we never insert reserved entries into the VIEP(RO).
+     return CodeOffsets::Verified_Inline_Entry_RO;
+   } else {
+     // Share same entry for VEP and VIEP(RO).
+     return CodeOffsets::Verified_Entry;
+   }
+ }
+ 
+ void CompiledEntrySignature::compute_calling_conventions() {
+   // Get the (non-scalarized) signature and check for inline type arguments
+   if (_method != NULL) {
+     if (!_method->is_static()) {
+       if (_method->method_holder()->is_inline_klass() && InlineKlass::cast(_method->method_holder())->can_be_passed_as_fields()) {
+         _has_inline_recv = true;
+         _num_inline_args++;
+       }
+       SigEntry::add_entry(_sig, T_OBJECT, _method->name());
+     }
+     for (SignatureStream ss(_method->signature()); !ss.at_return_type(); ss.next()) {
+       BasicType bt = ss.type();
+       if (bt == T_INLINE_TYPE) {
+         if (ss.as_inline_klass(_method->method_holder())->can_be_passed_as_fields()) {
+           _num_inline_args++;
+         }
+         bt = T_OBJECT;
+       }
+       SigEntry::add_entry(_sig, bt, ss.as_symbol());
+     }
+     if (_method->is_abstract() && !has_inline_arg()) {
+       return;
+     }
+   }
  
-   friend class SignatureIterator;  // so do_parameters_on can call do_type
-   void do_type(BasicType type) {
-     sig_bt[index++] = type;
-     if (type == T_LONG || type == T_DOUBLE) {
-       sig_bt[index++] = T_VOID; // Longs & doubles take 2 Java slots
+   // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage
+   _regs = NEW_RESOURCE_ARRAY(VMRegPair, _sig->length());
+   _args_on_stack = SharedRuntime::java_calling_convention(_sig, _regs);
+ 
+   // Now compute the scalarized calling convention if there are inline types in the signature
+   _regs_cc = _regs;
+   _regs_cc_ro = _regs;
+   _args_on_stack_cc = _args_on_stack;
+   _args_on_stack_cc_ro = _args_on_stack;
+ 
+   if (has_inline_arg() && !_method->is_native()) {
+     _args_on_stack_cc = compute_scalarized_cc(_sig_cc, _regs_cc, /* scalar_receiver = */ true);
+ 
+     _sig_cc_ro = _sig_cc;
+     _regs_cc_ro = _regs_cc;
+     _args_on_stack_cc_ro = _args_on_stack_cc;
+     if (_has_inline_recv) {
+       // For interface calls, we need another entry point / adapter to unpack the receiver
+       _args_on_stack_cc_ro = compute_scalarized_cc(_sig_cc_ro, _regs_cc_ro, /* scalar_receiver = */ false);
+     }
+ 
+     // Upper bound on stack arguments to avoid hitting the argument limit and
+     // bailing out of compilation ("unsupported incoming calling sequence").
+     // TODO we need a reasonable limit (flag?) here
+     if (_args_on_stack_cc > 50) {
+       // Don't scalarize inline type arguments
+       _sig_cc = _sig;
+       _sig_cc_ro = _sig;
+       _regs_cc = _regs;
+       _regs_cc_ro = _regs;
+       _args_on_stack_cc = _args_on_stack;
+       _args_on_stack_cc_ro = _args_on_stack;
+     } else {
+       _c1_needs_stack_repair = (_args_on_stack_cc < _args_on_stack) || (_args_on_stack_cc_ro < _args_on_stack);
+       _c2_needs_stack_repair = (_args_on_stack_cc > _args_on_stack) || (_args_on_stack_cc > _args_on_stack_cc_ro);
      }
    }
- };
+ }
  
  AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& method) {
    // Use customized signature handler.  Need to lock around updates to
    // the AdapterHandlerTable (it is not safe for concurrent readers
    // and a single writer: this could be fixed if it becomes a

@@ -2822,89 +3035,110 @@
    }
  
    ResourceMark rm;
    AdapterBlob* new_adapter = NULL;
  
-   // Fill in the signature array, for the calling-convention call.
-   int total_args_passed = method->size_of_parameters(); // All args on stack
+   CompiledEntrySignature ces(method());
+   ces.compute_calling_conventions();
+   if (ces.has_scalarized_args()) {
+     method->set_has_scalarized_args(true);
+     method->set_c1_needs_stack_repair(ces.c1_needs_stack_repair());
+     method->set_c2_needs_stack_repair(ces.c2_needs_stack_repair());
+   } else if (method->is_abstract()) {
+     return _abstract_method_handler;
+   }
  
-   AdapterSignatureIterator si(method->signature(), method->constMethod()->fingerprint(),
-                               method->is_static(), total_args_passed);
-   assert(si.slots() == total_args_passed, "");
-   BasicType* sig_bt = si.basic_types();
    {
      MutexLocker mu(AdapterHandlerLibrary_lock);
  
+     if (ces.has_scalarized_args() && method->is_abstract()) {
+       // Save a C heap allocated version of the signature for abstract methods with scalarized inline type arguments
+       address wrong_method_abstract = SharedRuntime::get_handle_wrong_method_abstract_stub();
+       entry = AdapterHandlerLibrary::new_entry(new AdapterFingerPrint(NULL),
+                                                StubRoutines::throw_AbstractMethodError_entry(),
+                                                wrong_method_abstract, wrong_method_abstract, wrong_method_abstract,
+                                                wrong_method_abstract, wrong_method_abstract);
+       GrowableArray<SigEntry>* heap_sig = new (ResourceObj::C_HEAP, mtInternal) GrowableArray<SigEntry>(ces.sig_cc_ro().length(), mtInternal);
+       heap_sig->appendAll(&ces.sig_cc_ro());
+       entry->set_sig_cc(heap_sig);
+       return entry;
+     }
+ 
      // Lookup method signature's fingerprint
-     entry = _adapters->lookup(total_args_passed, sig_bt);
+     entry = _adapters->lookup(&ces.sig_cc(), ces.regs_cc() != ces.regs_cc_ro());
  
      if (entry != NULL) {
  #ifdef ASSERT
        if (VerifyAdapterSharing) {
          AdapterBlob* comparison_blob = NULL;
-         AdapterHandlerEntry* comparison_entry = create_adapter(comparison_blob, total_args_passed, sig_bt, false);
+         AdapterHandlerEntry* comparison_entry = create_adapter(comparison_blob, ces, false);
          assert(comparison_blob == NULL, "no blob should be created when creating an adapter for comparison");
          assert(comparison_entry->compare_code(entry), "code must match");
          // Release the one just created and return the original
          _adapters->free_entry(comparison_entry);
        }
  #endif
        return entry;
      }
  
-     entry = create_adapter(new_adapter, total_args_passed, sig_bt, /* allocate_code_blob */ true);
+     entry = create_adapter(new_adapter, ces, /* allocate_code_blob */ true);
    }
  
    // Outside of the lock
    if (new_adapter != NULL) {
      post_adapter_creation(new_adapter, entry);
    }
    return entry;
  }
  
  AdapterHandlerEntry* AdapterHandlerLibrary::create_adapter(AdapterBlob*& new_adapter,
-                                                            int total_args_passed,
-                                                            BasicType* sig_bt,
+                                                            CompiledEntrySignature& ces,
                                                             bool allocate_code_blob) {
  
    // StubRoutines::code2() is initialized after this function can be called. As a result,
    // VerifyAdapterCalls and VerifyAdapterSharing can fail if we re-use code that generated
    // prior to StubRoutines::code2() being set. Checks refer to checks generated in an I2C
    // stub that ensure that an I2C stub is called from an interpreter frame.
    bool contains_all_checks = StubRoutines::code2() != NULL;
  
-   VMRegPair stack_regs[16];
-   VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed);
- 
-   // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage
-   int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed);
    BufferBlob* buf = buffer_blob(); // the temporary code buffer in CodeCache
    CodeBuffer buffer(buf);
    short buffer_locs[20];
    buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
                                            sizeof(buffer_locs)/sizeof(relocInfo));
  
    // Make a C heap allocated version of the fingerprint to store in the adapter
-   AdapterFingerPrint* fingerprint = new AdapterFingerPrint(total_args_passed, sig_bt);
+   AdapterFingerPrint* fingerprint = new AdapterFingerPrint(&ces.sig_cc(), ces.regs_cc() != ces.regs_cc_ro());
    MacroAssembler _masm(&buffer);
    AdapterHandlerEntry* entry = SharedRuntime::generate_i2c2i_adapters(&_masm,
-                                                 total_args_passed,
-                                                 comp_args_on_stack,
-                                                 sig_bt,
-                                                 regs,
-                                                 fingerprint);
+                                                 ces.args_on_stack(),
+                                                 &ces.sig(),
+                                                 ces.regs(),
+                                                 &ces.sig_cc(),
+                                                 ces.regs_cc(),
+                                                 &ces.sig_cc_ro(),
+                                                 ces.regs_cc_ro(),
+                                                 fingerprint,
+                                                 new_adapter,
+                                                 allocate_code_blob);
+ 
+   if (ces.has_scalarized_args()) {
+     // Save a C heap allocated version of the scalarized signature and store it in the adapter
+     GrowableArray<SigEntry>* heap_sig = new (ResourceObj::C_HEAP, mtInternal) GrowableArray<SigEntry>(ces.sig_cc().length(), mtInternal);
+     heap_sig->appendAll(&ces.sig_cc());
+     entry->set_sig_cc(heap_sig);
+   }
  
  #ifdef ASSERT
    if (VerifyAdapterSharing) {
      entry->save_code(buf->code_begin(), buffer.insts_size());
      if (!allocate_code_blob) {
        return entry;
      }
    }
  #endif
  
-   new_adapter = AdapterBlob::create(&buffer);
    NOT_PRODUCT(int insts_size = buffer.insts_size());
    if (new_adapter == NULL) {
      // CodeCache is full, disable compilation
      // Ought to log this but compile log is only per compile thread
      // and we're some non descript Java thread.

@@ -2941,11 +3175,14 @@
  
  address AdapterHandlerEntry::base_address() {
    address base = _i2c_entry;
    if (base == NULL)  base = _c2i_entry;
    assert(base <= _c2i_entry || _c2i_entry == NULL, "");
+   assert(base <= _c2i_inline_entry || _c2i_inline_entry == NULL, "");
+   assert(base <= _c2i_inline_ro_entry || _c2i_inline_ro_entry == NULL, "");
    assert(base <= _c2i_unverified_entry || _c2i_unverified_entry == NULL, "");
+   assert(base <= _c2i_unverified_inline_entry || _c2i_unverified_inline_entry == NULL, "");
    assert(base <= _c2i_no_clinit_check_entry || _c2i_no_clinit_check_entry == NULL, "");
    return base;
  }
  
  void AdapterHandlerEntry::relocate(address new_base) {

@@ -2954,20 +3191,29 @@
    ptrdiff_t delta = new_base - old_base;
    if (_i2c_entry != NULL)
      _i2c_entry += delta;
    if (_c2i_entry != NULL)
      _c2i_entry += delta;
+   if (_c2i_inline_entry != NULL)
+     _c2i_inline_entry += delta;
+   if (_c2i_inline_ro_entry != NULL)
+     _c2i_inline_ro_entry += delta;
    if (_c2i_unverified_entry != NULL)
      _c2i_unverified_entry += delta;
+   if (_c2i_unverified_inline_entry != NULL)
+     _c2i_unverified_inline_entry += delta;
    if (_c2i_no_clinit_check_entry != NULL)
      _c2i_no_clinit_check_entry += delta;
    assert(base_address() == new_base, "");
  }
  
  
  void AdapterHandlerEntry::deallocate() {
    delete _fingerprint;
+   if (_sig_cc != NULL) {
+     delete _sig_cc;
+   }
  #ifdef ASSERT
    FREE_C_HEAP_ARRAY(unsigned char, _saved_code);
  #endif
  }
  

@@ -3042,18 +3288,28 @@
        MacroAssembler _masm(&buffer);
  
        // Fill in the signature array, for the calling-convention call.
        const int total_args_passed = method->size_of_parameters();
  
+       BasicType stack_sig_bt[16];
        VMRegPair stack_regs[16];
+       BasicType* sig_bt = (total_args_passed <= 16) ? stack_sig_bt : NEW_RESOURCE_ARRAY(BasicType, total_args_passed);
        VMRegPair* regs = (total_args_passed <= 16) ? stack_regs : NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed);
  
-       AdapterSignatureIterator si(method->signature(), method->constMethod()->fingerprint(),
-                               method->is_static(), total_args_passed);
-       BasicType* sig_bt = si.basic_types();
-       assert(si.slots() == total_args_passed, "");
-       BasicType ret_type = si.return_type();
+       int i = 0;
+       if (!method->is_static()) {  // Pass in receiver first
+         sig_bt[i++] = T_OBJECT;
+       }
+       SignatureStream ss(method->signature());
+       for (; !ss.at_return_type(); ss.next()) {
+         sig_bt[i++] = ss.type();  // Collect remaining bits of signature
+         if (ss.type() == T_LONG || ss.type() == T_DOUBLE) {
+           sig_bt[i++] = T_VOID;   // Longs & doubles take 2 Java slots
+         }
+       }
+       assert(i == total_args_passed, "");
+       BasicType ret_type = ss.type();
  
        // Now get the compiled-Java arguments layout.
        int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed);
  
        // Generate the compiled-to-native wrapper code

@@ -3270,12 +3526,21 @@
      st->print(" i2c: " INTPTR_FORMAT, p2i(get_i2c_entry()));
    }
    if (get_c2i_entry() != NULL) {
      st->print(" c2i: " INTPTR_FORMAT, p2i(get_c2i_entry()));
    }
+   if (get_c2i_entry() != NULL) {
+     st->print(" c2iVE: " INTPTR_FORMAT, p2i(get_c2i_inline_entry()));
+   }
+   if (get_c2i_entry() != NULL) {
+     st->print(" c2iVROE: " INTPTR_FORMAT, p2i(get_c2i_inline_ro_entry()));
+   }
+   if (get_c2i_unverified_entry() != NULL) {
+     st->print(" c2iUE: " INTPTR_FORMAT, p2i(get_c2i_unverified_entry()));
+   }
    if (get_c2i_unverified_entry() != NULL) {
-     st->print(" c2iUV: " INTPTR_FORMAT, p2i(get_c2i_unverified_entry()));
+     st->print(" c2iUVE: " INTPTR_FORMAT, p2i(get_c2i_unverified_inline_entry()));
    }
    if (get_c2i_no_clinit_check_entry() != NULL) {
      st->print(" c2iNCI: " INTPTR_FORMAT, p2i(get_c2i_no_clinit_check_entry()));
    }
    st->cr();

@@ -3356,5 +3621,186 @@
    if (new_obj == NULL) return;
  
    BarrierSet *bs = BarrierSet::barrier_set();
    bs->on_slowpath_allocation_exit(current, new_obj);
  }
+ 
+ // We are at a compiled code to interpreter call. We need backing
+ // buffers for all inline type arguments. Allocate an object array to
+ // hold them (convenient because once we're done with it we don't have
+ // to worry about freeing it).
+ oop SharedRuntime::allocate_inline_types_impl(JavaThread* current, methodHandle callee, bool allocate_receiver, TRAPS) {
+   assert(InlineTypePassFieldsAsArgs, "no reason to call this");
+   ResourceMark rm;
+ 
+   int nb_slots = 0;
+   InstanceKlass* holder = callee->method_holder();
+   allocate_receiver &= !callee->is_static() && holder->is_inline_klass();
+   if (allocate_receiver) {
+     nb_slots++;
+   }
+   for (SignatureStream ss(callee->signature()); !ss.at_return_type(); ss.next()) {
+     if (ss.type() == T_INLINE_TYPE) {
+       nb_slots++;
+     }
+   }
+   objArrayOop array_oop = oopFactory::new_objectArray(nb_slots, CHECK_NULL);
+   objArrayHandle array(THREAD, array_oop);
+   int i = 0;
+   if (allocate_receiver) {
+     InlineKlass* vk = InlineKlass::cast(holder);
+     oop res = vk->allocate_instance(CHECK_NULL);
+     array->obj_at_put(i, res);
+     i++;
+   }
+   for (SignatureStream ss(callee->signature()); !ss.at_return_type(); ss.next()) {
+     if (ss.type() == T_INLINE_TYPE) {
+       InlineKlass* vk = ss.as_inline_klass(holder);
+       oop res = vk->allocate_instance(CHECK_NULL);
+       array->obj_at_put(i, res);
+       i++;
+     }
+   }
+   return array();
+ }
+ 
+ JRT_ENTRY(void, SharedRuntime::allocate_inline_types(JavaThread* current, Method* callee_method, bool allocate_receiver))
+   methodHandle callee(current, callee_method);
+   oop array = SharedRuntime::allocate_inline_types_impl(current, callee, allocate_receiver, CHECK);
+   current->set_vm_result(array);
+   current->set_vm_result_2(callee()); // TODO: required to keep callee live?
+ JRT_END
+ 
+ // We're returning from an interpreted method: load each field into a
+ // register following the calling convention
+ JRT_LEAF(void, SharedRuntime::load_inline_type_fields_in_regs(JavaThread* current, oopDesc* res))
+ {
+   assert(res->klass()->is_inline_klass(), "only inline types here");
+   ResourceMark rm;
+   RegisterMap reg_map(current);
+   frame stubFrame = current->last_frame();
+   frame callerFrame = stubFrame.sender(&reg_map);
+   assert(callerFrame.is_interpreted_frame(), "should be coming from interpreter");
+ 
+   InlineKlass* vk = InlineKlass::cast(res->klass());
+ 
+   const Array<SigEntry>* sig_vk = vk->extended_sig();
+   const Array<VMRegPair>* regs = vk->return_regs();
+ 
+   if (regs == NULL) {
+     // The fields of the inline klass don't fit in registers, bail out
+     return;
+   }
+ 
+   int j = 1;
+   for (int i = 0; i < sig_vk->length(); i++) {
+     BasicType bt = sig_vk->at(i)._bt;
+     if (bt == T_INLINE_TYPE) {
+       continue;
+     }
+     if (bt == T_VOID) {
+       if (sig_vk->at(i-1)._bt == T_LONG ||
+           sig_vk->at(i-1)._bt == T_DOUBLE) {
+         j++;
+       }
+       continue;
+     }
+     int off = sig_vk->at(i)._offset;
+     assert(off > 0, "offset in object should be positive");
+     VMRegPair pair = regs->at(j);
+     address loc = reg_map.location(pair.first());
+     switch(bt) {
+     case T_BOOLEAN:
+       *(jboolean*)loc = res->bool_field(off);
+       break;
+     case T_CHAR:
+       *(jchar*)loc = res->char_field(off);
+       break;
+     case T_BYTE:
+       *(jbyte*)loc = res->byte_field(off);
+       break;
+     case T_SHORT:
+       *(jshort*)loc = res->short_field(off);
+       break;
+     case T_INT: {
+       *(jint*)loc = res->int_field(off);
+       break;
+     }
+     case T_LONG:
+ #ifdef _LP64
+       *(intptr_t*)loc = res->long_field(off);
+ #else
+       Unimplemented();
+ #endif
+       break;
+     case T_OBJECT:
+     case T_ARRAY: {
+       *(oop*)loc = res->obj_field(off);
+       break;
+     }
+     case T_FLOAT:
+       *(jfloat*)loc = res->float_field(off);
+       break;
+     case T_DOUBLE:
+       *(jdouble*)loc = res->double_field(off);
+       break;
+     default:
+       ShouldNotReachHere();
+     }
+     j++;
+   }
+   assert(j == regs->length(), "missed a field?");
+ 
+ #ifdef ASSERT
+   VMRegPair pair = regs->at(0);
+   address loc = reg_map.location(pair.first());
+   assert(*(oopDesc**)loc == res, "overwritten object");
+ #endif
+ 
+   current->set_vm_result(res);
+ }
+ JRT_END
+ 
+ // We've returned to an interpreted method, the interpreter needs a
+ // reference to an inline type instance. Allocate it and initialize it
+ // from field's values in registers.
+ JRT_BLOCK_ENTRY(void, SharedRuntime::store_inline_type_fields_to_buf(JavaThread* current, intptr_t res))
+ {
+   ResourceMark rm;
+   RegisterMap reg_map(current);
+   frame stubFrame = current->last_frame();
+   frame callerFrame = stubFrame.sender(&reg_map);
+ 
+ #ifdef ASSERT
+   InlineKlass* verif_vk = InlineKlass::returned_inline_klass(reg_map);
+ #endif
+ 
+   if (!is_set_nth_bit(res, 0)) {
+     // We're not returning with inline type fields in registers (the
+     // calling convention didn't allow it for this inline klass)
+     assert(!Metaspace::contains((void*)res), "should be oop or pointer in buffer area");
+     current->set_vm_result((oopDesc*)res);
+     assert(verif_vk == NULL, "broken calling convention");
+     return;
+   }
+ 
+   clear_nth_bit(res, 0);
+   InlineKlass* vk = (InlineKlass*)res;
+   assert(verif_vk == vk, "broken calling convention");
+   assert(Metaspace::contains((void*)res), "should be klass");
+ 
+   // Allocate handles for every oop field so they are safe in case of
+   // a safepoint when allocating
+   GrowableArray<Handle> handles;
+   vk->save_oop_fields(reg_map, handles);
+ 
+   // It's unsafe to safepoint until we are here
+   JRT_BLOCK;
+   {
+     JavaThread* THREAD = current;
+     oop vt = vk->realloc_result(reg_map, handles, CHECK);
+     current->set_vm_result(vt);
+   }
+   JRT_BLOCK_END;
+ }
+ JRT_END
+ 
< prev index next >