1 /*
   2  * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "ci/bcEscapeAnalyzer.hpp"
  27 #include "ci/ciCallSite.hpp"
  28 #include "ci/ciObjArray.hpp"
  29 #include "ci/ciMemberName.hpp"
  30 #include "ci/ciMethodHandle.hpp"
  31 #include "classfile/javaClasses.hpp"
  32 #include "compiler/compileLog.hpp"
  33 #include "opto/addnode.hpp"
  34 #include "opto/callGenerator.hpp"
  35 #include "opto/callnode.hpp"
  36 #include "opto/castnode.hpp"
  37 #include "opto/cfgnode.hpp"
  38 #include "opto/inlinetypenode.hpp"
  39 #include "opto/parse.hpp"
  40 #include "opto/rootnode.hpp"
  41 #include "opto/runtime.hpp"
  42 #include "opto/subnode.hpp"
  43 #include "runtime/sharedRuntime.hpp"
  44 #include "ci/ciNativeEntryPoint.hpp"
  45 #include "utilities/debug.hpp"
  46 
  47 // Utility function.
  48 const TypeFunc* CallGenerator::tf() const {
  49   return TypeFunc::make(method());
  50 }
  51 
  52 bool CallGenerator::is_inlined_method_handle_intrinsic(JVMState* jvms, ciMethod* m) {
  53   return is_inlined_method_handle_intrinsic(jvms->method(), jvms->bci(), m);
  54 }
  55 
  56 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* caller, int bci, ciMethod* m) {
  57   ciMethod* symbolic_info = caller->get_method_at_bci(bci);
  58   return is_inlined_method_handle_intrinsic(symbolic_info, m);
  59 }
  60 
  61 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* symbolic_info, ciMethod* m) {
  62   return symbolic_info->is_method_handle_intrinsic() && !m->is_method_handle_intrinsic();
  63 }
  64 
  65 //-----------------------------ParseGenerator---------------------------------
  66 // Internal class which handles all direct bytecode traversal.
  67 class ParseGenerator : public InlineCallGenerator {
  68 private:
  69   bool  _is_osr;
  70   float _expected_uses;
  71 
  72 public:
  73   ParseGenerator(ciMethod* method, float expected_uses, bool is_osr = false)
  74     : InlineCallGenerator(method)
  75   {
  76     _is_osr        = is_osr;
  77     _expected_uses = expected_uses;
  78     assert(InlineTree::check_can_parse(method) == NULL, "parse must be possible");
  79   }
  80 
  81   virtual bool      is_parse() const           { return true; }
  82   virtual JVMState* generate(JVMState* jvms);
  83   int is_osr() { return _is_osr; }
  84 
  85 };
  86 
  87 JVMState* ParseGenerator::generate(JVMState* jvms) {
  88   Compile* C = Compile::current();
  89   C->print_inlining_update(this);
  90 
  91   if (is_osr()) {
  92     // The JVMS for a OSR has a single argument (see its TypeFunc).
  93     assert(jvms->depth() == 1, "no inline OSR");
  94   }
  95 
  96   if (C->failing()) {
  97     return NULL;  // bailing out of the compile; do not try to parse
  98   }
  99 
 100   Parse parser(jvms, method(), _expected_uses);
 101   // Grab signature for matching/allocation
 102   GraphKit& exits = parser.exits();
 103 
 104   if (C->failing()) {
 105     while (exits.pop_exception_state() != NULL) ;
 106     return NULL;
 107   }
 108 
 109   assert(exits.jvms()->same_calls_as(jvms), "sanity");
 110 
 111   // Simply return the exit state of the parser,
 112   // augmented by any exceptional states.
 113   return exits.transfer_exceptions_into_jvms();
 114 }
 115 
 116 //---------------------------DirectCallGenerator------------------------------
 117 // Internal class which handles all out-of-line calls w/o receiver type checks.
 118 class DirectCallGenerator : public CallGenerator {
 119  private:
 120   CallStaticJavaNode* _call_node;
 121   // Force separate memory and I/O projections for the exceptional
 122   // paths to facilitate late inlining.
 123   bool                _separate_io_proj;
 124 
 125 protected:
 126   void set_call_node(CallStaticJavaNode* call) { _call_node = call; }
 127 
 128  public:
 129   DirectCallGenerator(ciMethod* method, bool separate_io_proj)
 130     : CallGenerator(method),
 131       _call_node(NULL),
 132       _separate_io_proj(separate_io_proj)
 133   {
 134     if (InlineTypeReturnedAsFields && method->is_method_handle_intrinsic()) {
 135       // If that call has not been optimized by the time optimizations are over,
 136       // we'll need to add a call to create an inline type instance from the klass
 137       // returned by the call (see PhaseMacroExpand::expand_mh_intrinsic_return).
 138       // Separating memory and I/O projections for exceptions is required to
 139       // perform that graph transformation.
 140       _separate_io_proj = true;
 141     }
 142   }
 143   virtual JVMState* generate(JVMState* jvms);
 144 
 145   virtual CallNode* call_node() const { return _call_node; }
 146   virtual CallGenerator* with_call_node(CallNode* call) {
 147     DirectCallGenerator* dcg = new DirectCallGenerator(method(), _separate_io_proj);
 148     dcg->set_call_node(call->as_CallStaticJava());
 149     return dcg;
 150   }
 151 };
 152 
 153 JVMState* DirectCallGenerator::generate(JVMState* jvms) {
 154   GraphKit kit(jvms);
 155   kit.C->print_inlining_update(this);
 156   PhaseGVN& gvn = kit.gvn();
 157   bool is_static = method()->is_static();
 158   address target = is_static ? SharedRuntime::get_resolve_static_call_stub()
 159                              : SharedRuntime::get_resolve_opt_virtual_call_stub();
 160 
 161   if (kit.C->log() != NULL) {
 162     kit.C->log()->elem("direct_call bci='%d'", jvms->bci());
 163   }
 164 
 165   CallStaticJavaNode* call = new CallStaticJavaNode(kit.C, tf(), target, method());
 166   if (is_inlined_method_handle_intrinsic(jvms, method())) {
 167     // To be able to issue a direct call and skip a call to MH.linkTo*/invokeBasic adapter,
 168     // additional information about the method being invoked should be attached
 169     // to the call site to make resolution logic work
 170     // (see SharedRuntime::resolve_static_call_C).
 171     call->set_override_symbolic_info(true);
 172   }
 173   _call_node = call;  // Save the call node in case we need it later
 174   if (!is_static) {
 175     // Make an explicit receiver null_check as part of this call.
 176     // Since we share a map with the caller, his JVMS gets adjusted.
 177     kit.null_check_receiver_before_call(method());
 178     if (kit.stopped()) {
 179       // And dump it back to the caller, decorated with any exceptions:
 180       return kit.transfer_exceptions_into_jvms();
 181     }
 182     // Mark the call node as virtual, sort of:
 183     call->set_optimized_virtual(true);
 184     if (method()->is_method_handle_intrinsic() ||
 185         method()->is_compiled_lambda_form()) {
 186       call->set_method_handle_invoke(true);
 187     }
 188   }
 189   kit.set_arguments_for_java_call(call, is_late_inline());
 190   if (kit.stopped()) {
 191     return kit.transfer_exceptions_into_jvms();
 192   }
 193   kit.set_edges_for_java_call(call, false, _separate_io_proj);
 194   Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
 195   kit.push_node(method()->return_type()->basic_type(), ret);
 196   return kit.transfer_exceptions_into_jvms();
 197 }
 198 
 199 //--------------------------VirtualCallGenerator------------------------------
 200 // Internal class which handles all out-of-line calls checking receiver type.
 201 class VirtualCallGenerator : public CallGenerator {
 202 private:
 203   int _vtable_index;
 204   bool _separate_io_proj;
 205   CallDynamicJavaNode* _call_node;
 206 
 207 protected:
 208   void set_call_node(CallDynamicJavaNode* call) { _call_node = call; }
 209 
 210 public:
 211   VirtualCallGenerator(ciMethod* method, int vtable_index, bool separate_io_proj)
 212     : CallGenerator(method), _vtable_index(vtable_index), _separate_io_proj(separate_io_proj), _call_node(NULL)
 213   {
 214     assert(vtable_index == Method::invalid_vtable_index ||
 215            vtable_index >= 0, "either invalid or usable");
 216   }
 217   virtual bool      is_virtual() const          { return true; }
 218   virtual JVMState* generate(JVMState* jvms);
 219 
 220   virtual CallNode* call_node() const { return _call_node; }
 221   int vtable_index() const { return _vtable_index; }
 222 
 223   virtual CallGenerator* with_call_node(CallNode* call) {
 224     VirtualCallGenerator* cg = new VirtualCallGenerator(method(), _vtable_index, _separate_io_proj);
 225     cg->set_call_node(call->as_CallDynamicJava());
 226     return cg;
 227   }
 228 };
 229 
 230 JVMState* VirtualCallGenerator::generate(JVMState* jvms) {
 231   GraphKit kit(jvms);
 232   Node* receiver = kit.argument(0);
 233   kit.C->print_inlining_update(this);
 234 
 235   if (kit.C->log() != NULL) {
 236     kit.C->log()->elem("virtual_call bci='%d'", jvms->bci());
 237   }
 238 
 239   // If the receiver is a constant null, do not torture the system
 240   // by attempting to call through it.  The compile will proceed
 241   // correctly, but may bail out in final_graph_reshaping, because
 242   // the call instruction will have a seemingly deficient out-count.
 243   // (The bailout says something misleading about an "infinite loop".)
 244   if (!receiver->is_InlineType() && kit.gvn().type(receiver)->higher_equal(TypePtr::NULL_PTR)) {
 245     assert(Bytecodes::is_invoke(kit.java_bc()), "%d: %s", kit.java_bc(), Bytecodes::name(kit.java_bc()));
 246     ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci());
 247     int arg_size = declared_method->signature()->arg_size_for_bc(kit.java_bc());
 248     kit.inc_sp(arg_size);  // restore arguments
 249     kit.uncommon_trap(Deoptimization::Reason_null_check,
 250                       Deoptimization::Action_none,
 251                       NULL, "null receiver");
 252     return kit.transfer_exceptions_into_jvms();
 253   }
 254 
 255   // Ideally we would unconditionally do a null check here and let it
 256   // be converted to an implicit check based on profile information.
 257   // However currently the conversion to implicit null checks in
 258   // Block::implicit_null_check() only looks for loads and stores, not calls.
 259   ciMethod *caller = kit.method();
 260   ciMethodData *caller_md = (caller == NULL) ? NULL : caller->method_data();
 261   if (!UseInlineCaches || !ImplicitNullChecks || !os::zero_page_read_protected() ||
 262        ((ImplicitNullCheckThreshold > 0) && caller_md &&
 263        (caller_md->trap_count(Deoptimization::Reason_null_check)
 264        >= (uint)ImplicitNullCheckThreshold))) {
 265     // Make an explicit receiver null_check as part of this call.
 266     // Since we share a map with the caller, his JVMS gets adjusted.
 267     receiver = kit.null_check_receiver_before_call(method());
 268     if (kit.stopped()) {
 269       // And dump it back to the caller, decorated with any exceptions:
 270       return kit.transfer_exceptions_into_jvms();
 271     }
 272   }
 273 
 274   assert(!method()->is_static(), "virtual call must not be to static");
 275   assert(!method()->is_final(), "virtual call should not be to final");
 276   assert(!method()->is_private(), "virtual call should not be to private");
 277   assert(_vtable_index == Method::invalid_vtable_index || !UseInlineCaches,
 278          "no vtable calls if +UseInlineCaches ");
 279   address target = SharedRuntime::get_resolve_virtual_call_stub();
 280   // Normal inline cache used for call
 281   CallDynamicJavaNode* call = new CallDynamicJavaNode(tf(), target, method(), _vtable_index);
 282   if (is_inlined_method_handle_intrinsic(jvms, method())) {
 283     // To be able to issue a direct call (optimized virtual or virtual)
 284     // and skip a call to MH.linkTo*/invokeBasic adapter, additional information
 285     // about the method being invoked should be attached to the call site to
 286     // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C).
 287     call->set_override_symbolic_info(true);
 288   }
 289   _call_node = call;  // Save the call node in case we need it later
 290 
 291   kit.set_arguments_for_java_call(call);
 292   if (kit.stopped()) {
 293     return kit.transfer_exceptions_into_jvms();
 294   }
 295   kit.set_edges_for_java_call(call, false /*must_throw*/, _separate_io_proj);
 296   Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
 297   kit.push_node(method()->return_type()->basic_type(), ret);
 298 
 299   // Represent the effect of an implicit receiver null_check
 300   // as part of this call.  Since we share a map with the caller,
 301   // his JVMS gets adjusted.
 302   kit.cast_not_null(receiver);
 303   return kit.transfer_exceptions_into_jvms();
 304 }
 305 
 306 CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) {
 307   if (InlineTree::check_can_parse(m) != NULL)  return NULL;
 308   return new ParseGenerator(m, expected_uses);
 309 }
 310 
 311 // As a special case, the JVMS passed to this CallGenerator is
 312 // for the method execution already in progress, not just the JVMS
 313 // of the caller.  Thus, this CallGenerator cannot be mixed with others!
 314 CallGenerator* CallGenerator::for_osr(ciMethod* m, int osr_bci) {
 315   if (InlineTree::check_can_parse(m) != NULL)  return NULL;
 316   float past_uses = m->interpreter_invocation_count();
 317   float expected_uses = past_uses;
 318   return new ParseGenerator(m, expected_uses, true);
 319 }
 320 
 321 CallGenerator* CallGenerator::for_direct_call(ciMethod* m, bool separate_io_proj) {
 322   assert(!m->is_abstract(), "for_direct_call mismatch");
 323   return new DirectCallGenerator(m, separate_io_proj);
 324 }
 325 
 326 CallGenerator* CallGenerator::for_virtual_call(ciMethod* m, int vtable_index) {
 327   assert(!m->is_static(), "for_virtual_call mismatch");
 328   assert(!m->is_method_handle_intrinsic(), "should be a direct call");
 329   return new VirtualCallGenerator(m, vtable_index, false /*separate_io_projs*/);
 330 }
 331 
 332 // Allow inlining decisions to be delayed
 333 class LateInlineCallGenerator : public DirectCallGenerator {
 334  private:
 335   jlong _unique_id;   // unique id for log compilation
 336   bool _is_pure_call; // a hint that the call doesn't have important side effects to care about
 337 
 338  protected:
 339   CallGenerator* _inline_cg;
 340   virtual bool do_late_inline_check(Compile* C, JVMState* jvms) { return true; }
 341   virtual CallGenerator* inline_cg() const { return _inline_cg; }
 342   virtual bool is_pure_call() const { return _is_pure_call; }
 343 
 344  public:
 345   LateInlineCallGenerator(ciMethod* method, CallGenerator* inline_cg, bool is_pure_call = false) :
 346     DirectCallGenerator(method, true), _unique_id(0), _is_pure_call(is_pure_call), _inline_cg(inline_cg) {}
 347 
 348   virtual bool is_late_inline() const { return true; }
 349 
 350   // Convert the CallStaticJava into an inline
 351   virtual void do_late_inline();
 352 
 353   virtual JVMState* generate(JVMState* jvms) {
 354     Compile *C = Compile::current();
 355 
 356     C->log_inline_id(this);
 357 
 358     // Record that this call site should be revisited once the main
 359     // parse is finished.
 360     if (!is_mh_late_inline()) {
 361       C->add_late_inline(this);
 362     }
 363 
 364     // Emit the CallStaticJava and request separate projections so
 365     // that the late inlining logic can distinguish between fall
 366     // through and exceptional uses of the memory and io projections
 367     // as is done for allocations and macro expansion.
 368     return DirectCallGenerator::generate(jvms);
 369   }
 370 
 371   virtual void print_inlining_late(const char* msg) {
 372     CallNode* call = call_node();
 373     Compile* C = Compile::current();
 374     C->print_inlining_assert_ready();
 375     C->print_inlining(method(), call->jvms()->depth()-1, call->jvms()->bci(), msg);
 376     C->print_inlining_move_to(this);
 377     C->print_inlining_update_delayed(this);
 378   }
 379 
 380   virtual void set_unique_id(jlong id) {
 381     _unique_id = id;
 382   }
 383 
 384   virtual jlong unique_id() const {
 385     return _unique_id;
 386   }
 387 
 388   virtual CallGenerator* inline_cg() {
 389     return _inline_cg;
 390   }
 391 
 392   virtual CallGenerator* with_call_node(CallNode* call) {
 393     LateInlineCallGenerator* cg = new LateInlineCallGenerator(method(), _inline_cg, _is_pure_call);
 394     cg->set_call_node(call->as_CallStaticJava());
 395     return cg;
 396   }
 397 };
 398 
 399 CallGenerator* CallGenerator::for_late_inline(ciMethod* method, CallGenerator* inline_cg) {
 400   return new LateInlineCallGenerator(method, inline_cg);
 401 }
 402 
 403 class LateInlineMHCallGenerator : public LateInlineCallGenerator {
 404   ciMethod* _caller;
 405   bool _input_not_const;
 406 
 407   virtual bool do_late_inline_check(Compile* C, JVMState* jvms);
 408 
 409  public:
 410   LateInlineMHCallGenerator(ciMethod* caller, ciMethod* callee, bool input_not_const) :
 411     LateInlineCallGenerator(callee, NULL), _caller(caller), _input_not_const(input_not_const) {}
 412 
 413   virtual bool is_mh_late_inline() const { return true; }
 414 
 415   // Convert the CallStaticJava into an inline
 416   virtual void do_late_inline();
 417 
 418   virtual JVMState* generate(JVMState* jvms) {
 419     JVMState* new_jvms = LateInlineCallGenerator::generate(jvms);
 420 
 421     Compile* C = Compile::current();
 422     if (_input_not_const) {
 423       // inlining won't be possible so no need to enqueue right now.
 424       call_node()->set_generator(this);
 425     } else {
 426       C->add_late_inline(this);
 427     }
 428     return new_jvms;
 429   }
 430 
 431   virtual CallGenerator* with_call_node(CallNode* call) {
 432     LateInlineMHCallGenerator* cg = new LateInlineMHCallGenerator(_caller, method(), _input_not_const);
 433     cg->set_call_node(call->as_CallStaticJava());
 434     return cg;
 435   }
 436 };
 437 
 438 bool LateInlineMHCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) {
 439   // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call.
 440   bool allow_inline = C->inlining_incrementally();
 441   bool input_not_const = true;
 442   CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), allow_inline, input_not_const);
 443   assert(!input_not_const, "sanity"); // shouldn't have been scheduled for inlining in the first place
 444 
 445   if (cg != NULL) {
 446     // AlwaysIncrementalInline causes for_method_handle_inline() to
 447     // return a LateInlineCallGenerator. Extract the
 448     // InlineCallGenerato from it.
 449     if (AlwaysIncrementalInline && cg->is_late_inline()) {
 450       cg = cg->inline_cg();
 451     }
 452 
 453     assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline, "we're doing late inlining");
 454     _inline_cg = cg;
 455     C->dec_number_of_mh_late_inlines();
 456     return true;
 457   } else {
 458     // Method handle call which has a constant appendix argument should be either inlined or replaced with a direct call
 459     // unless there's a signature mismatch between caller and callee. If the failure occurs, there's not much to be improved later,
 460     // so don't reinstall the generator to avoid pushing the generator between IGVN and incremental inlining indefinitely.
 461     return false;
 462   }
 463 }
 464 
 465 CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const) {
 466   assert(IncrementalInlineMH, "required");
 467   Compile::current()->inc_number_of_mh_late_inlines();
 468   CallGenerator* cg = new LateInlineMHCallGenerator(caller, callee, input_not_const);
 469   return cg;
 470 }
 471 
 472 // Allow inlining decisions to be delayed
 473 class LateInlineVirtualCallGenerator : public VirtualCallGenerator {
 474  private:
 475   jlong          _unique_id;   // unique id for log compilation
 476   CallGenerator* _inline_cg;
 477   ciMethod*      _callee;
 478   bool           _is_pure_call;
 479   float          _prof_factor;
 480 
 481  protected:
 482   virtual bool do_late_inline_check(Compile* C, JVMState* jvms);
 483   virtual CallGenerator* inline_cg() const { return _inline_cg; }
 484   virtual bool is_pure_call() const { return _is_pure_call; }
 485 
 486  public:
 487   LateInlineVirtualCallGenerator(ciMethod* method, int vtable_index, float prof_factor)
 488   : VirtualCallGenerator(method, vtable_index, true /*separate_io_projs*/),
 489     _unique_id(0), _inline_cg(NULL), _callee(NULL), _is_pure_call(false), _prof_factor(prof_factor) {
 490     assert(IncrementalInlineVirtual, "required");
 491   }
 492 
 493   virtual bool is_late_inline() const { return true; }
 494 
 495   virtual bool is_virtual_late_inline() const { return true; }
 496 
 497   // Convert the CallDynamicJava into an inline
 498   virtual void do_late_inline();
 499 
 500   virtual void set_callee_method(ciMethod* m) {
 501     assert(_callee == NULL, "repeated inlining attempt");
 502     _callee = m;
 503   }
 504 
 505   virtual JVMState* generate(JVMState* jvms) {
 506     // Emit the CallDynamicJava and request separate projections so
 507     // that the late inlining logic can distinguish between fall
 508     // through and exceptional uses of the memory and io projections
 509     // as is done for allocations and macro expansion.
 510     JVMState* new_jvms = VirtualCallGenerator::generate(jvms);
 511     if (call_node() != NULL) {
 512       call_node()->set_generator(this);
 513     }
 514     return new_jvms;
 515   }
 516 
 517   virtual void print_inlining_late(const char* msg) {
 518     CallNode* call = call_node();
 519     Compile* C = Compile::current();
 520     C->print_inlining_assert_ready();
 521     C->print_inlining(method(), call->jvms()->depth()-1, call->jvms()->bci(), msg);
 522     C->print_inlining_move_to(this);
 523     C->print_inlining_update_delayed(this);
 524   }
 525 
 526   virtual void set_unique_id(jlong id) {
 527     _unique_id = id;
 528   }
 529 
 530   virtual jlong unique_id() const {
 531     return _unique_id;
 532   }
 533 
 534   virtual CallGenerator* with_call_node(CallNode* call) {
 535     LateInlineVirtualCallGenerator* cg = new LateInlineVirtualCallGenerator(method(), vtable_index(), _prof_factor);
 536     cg->set_call_node(call->as_CallDynamicJava());
 537     return cg;
 538   }
 539 };
 540 
 541 bool LateInlineVirtualCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) {
 542   // Method handle linker case is handled in CallDynamicJavaNode::Ideal().
 543   // Unless inlining is performed, _override_symbolic_info bit will be set in DirectCallGenerator::generate().
 544 
 545   // Implicit receiver null checks introduce problems when exception states are combined.
 546   Node* receiver = jvms->map()->argument(jvms, 0);
 547   const Type* recv_type = C->initial_gvn()->type(receiver);
 548   if (recv_type->maybe_null()) {
 549     return false;
 550   }
 551   // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call.
 552   bool allow_inline = C->inlining_incrementally();
 553   if (!allow_inline && _callee->holder()->is_interface()) {
 554     // Don't convert the interface call to a direct call guarded by an interface subtype check.
 555     return false;
 556   }
 557   CallGenerator* cg = C->call_generator(_callee,
 558                                         vtable_index(),
 559                                         false /*call_does_dispatch*/,
 560                                         jvms,
 561                                         allow_inline,
 562                                         _prof_factor,
 563                                         NULL /*speculative_receiver_type*/,
 564                                         true /*allow_intrinsics*/);
 565 
 566   if (cg != NULL) {
 567     assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline, "we're doing late inlining");
 568     _inline_cg = cg;
 569     return true;
 570   } else {
 571     // Virtual call which provably doesn't dispatch should be either inlined or replaced with a direct call.
 572     assert(false, "no progress");
 573     return false;
 574   }
 575 }
 576 
 577 CallGenerator* CallGenerator::for_late_inline_virtual(ciMethod* m, int vtable_index, float prof_factor) {
 578   assert(IncrementalInlineVirtual, "required");
 579   assert(!m->is_static(), "for_virtual_call mismatch");
 580   assert(!m->is_method_handle_intrinsic(), "should be a direct call");
 581   return new LateInlineVirtualCallGenerator(m, vtable_index, prof_factor);
 582 }
 583 
 584 void LateInlineCallGenerator::do_late_inline() {
 585   CallGenerator::do_late_inline_helper();
 586 }
 587 
 588 void LateInlineMHCallGenerator::do_late_inline() {
 589   CallGenerator::do_late_inline_helper();
 590 }
 591 
 592 void LateInlineVirtualCallGenerator::do_late_inline() {
 593   assert(_callee != NULL, "required"); // set up in CallDynamicJavaNode::Ideal
 594   CallGenerator::do_late_inline_helper();
 595 }
 596 
 597 static bool has_non_debug_usages(Node* n) {
 598   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
 599     Node* m = n->fast_out(i);
 600     if (!m->is_SafePoint()
 601         || (m->is_Call() && m->as_Call()->has_non_debug_use(n))) {
 602       return true;
 603     }
 604   }
 605   return false;
 606 }
 607 
 608 static bool is_box_cache_valid(CallNode* call) {
 609   ciInstanceKlass* klass = call->as_CallStaticJava()->method()->holder();
 610   return klass->is_box_cache_valid();
 611 }
 612 
 613 // delay box in runtime, treat box as a scalarized object
 614 static void scalarize_debug_usages(CallNode* call, Node* resproj) {
 615   GraphKit kit(call->jvms());
 616   PhaseGVN& gvn = kit.gvn();
 617 
 618   ProjNode* res = resproj->as_Proj();
 619   ciInstanceKlass* klass = call->as_CallStaticJava()->method()->holder();
 620   int n_fields = klass->nof_nonstatic_fields();
 621   assert(n_fields == 1, "the klass must be an auto-boxing klass");
 622 
 623   for (DUIterator_Last imin, i = res->last_outs(imin); i >= imin;) {
 624     SafePointNode* sfpt = res->last_out(i)->as_SafePoint();
 625     uint first_ind = sfpt->req() - sfpt->jvms()->scloff();
 626     Node* sobj = new SafePointScalarObjectNode(gvn.type(res)->isa_oopptr(),
 627 #ifdef ASSERT
 628                                                 call,
 629 #endif // ASSERT
 630                                                 first_ind, n_fields, true);
 631     sobj->init_req(0, kit.root());
 632     sfpt->add_req(call->in(TypeFunc::Parms));
 633     sobj = gvn.transform(sobj);
 634     JVMState* jvms = sfpt->jvms();
 635     jvms->set_endoff(sfpt->req());
 636     int start = jvms->debug_start();
 637     int end   = jvms->debug_end();
 638     int num_edges = sfpt->replace_edges_in_range(res, sobj, start, end, &gvn);
 639     i -= num_edges;
 640   }
 641 
 642   assert(res->outcnt() == 0, "the box must have no use after replace");
 643 
 644 #ifndef PRODUCT
 645   if (PrintEliminateAllocations) {
 646     tty->print("++++ Eliminated: %d ", call->_idx);
 647     call->as_CallStaticJava()->method()->print_short_name(tty);
 648     tty->cr();
 649   }
 650 #endif
 651 }
 652 
 653 void CallGenerator::do_late_inline_helper() {
 654   assert(is_late_inline(), "only late inline allowed");
 655 
 656   // Can't inline it
 657   CallNode* call = call_node();
 658   if (call == NULL || call->outcnt() == 0 ||
 659       call->in(0) == NULL || call->in(0)->is_top()) {
 660     return;
 661   }
 662 
 663   const TypeTuple* r = call->tf()->domain_cc();
 664   for (uint i1 = TypeFunc::Parms; i1 < r->cnt(); i1++) {
 665     if (call->in(i1)->is_top() && r->field_at(i1) != Type::HALF) {
 666       assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
 667       return;
 668     }
 669   }
 670 
 671   if (call->in(TypeFunc::Memory)->is_top()) {
 672     assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
 673     return;
 674   }
 675   if (call->in(TypeFunc::Memory)->is_MergeMem()) {
 676     MergeMemNode* merge_mem = call->in(TypeFunc::Memory)->as_MergeMem();
 677     if (merge_mem->base_memory() == merge_mem->empty_memory()) {
 678       return; // dead path
 679     }
 680   }
 681 
 682   // check for unreachable loop
 683   CallProjections* callprojs = call->extract_projections(true);
 684   if ((callprojs->fallthrough_catchproj == call->in(0)) ||
 685       (callprojs->catchall_catchproj    == call->in(0)) ||
 686       (callprojs->fallthrough_memproj   == call->in(TypeFunc::Memory)) ||
 687       (callprojs->catchall_memproj      == call->in(TypeFunc::Memory)) ||
 688       (callprojs->fallthrough_ioproj    == call->in(TypeFunc::I_O)) ||
 689       (callprojs->catchall_ioproj       == call->in(TypeFunc::I_O)) ||
 690       (callprojs->exobj != NULL && call->find_edge(callprojs->exobj) != -1)) {
 691     return;
 692   }
 693 
 694   Compile* C = Compile::current();
 695   // Remove inlined methods from Compiler's lists.
 696   if (call->is_macro()) {
 697     C->remove_macro_node(call);
 698   }
 699 
 700   bool result_not_used = false;
 701 
 702   if (is_pure_call()) {
 703     // Disabled due to JDK-8276112
 704     if (false && is_boxing_late_inline() && callprojs->resproj[0] != nullptr) {
 705         // replace box node to scalar node only in case it is directly referenced by debug info
 706         assert(call->as_CallStaticJava()->is_boxing_method(), "sanity");
 707         if (!has_non_debug_usages(callprojs->resproj[0]) && is_box_cache_valid(call)) {
 708           scalarize_debug_usages(call, callprojs->resproj[0]);
 709         }
 710     }
 711 
 712     // The call is marked as pure (no important side effects), but result isn't used.
 713     // It's safe to remove the call.
 714     result_not_used = true;
 715     for (uint i = 0; i < callprojs->nb_resproj; i++) {
 716       if (callprojs->resproj[i] != NULL) {
 717         if (callprojs->resproj[i]->outcnt() != 0) {
 718           result_not_used = false;
 719         }
 720         if (call->find_edge(callprojs->resproj[i]) != -1) {
 721           return;
 722         }
 723       }
 724     }
 725   }
 726 
 727   if (result_not_used) {
 728     GraphKit kit(call->jvms());
 729     kit.replace_call(call, C->top(), true);
 730   } else {
 731     // Make a clone of the JVMState that appropriate to use for driving a parse
 732     JVMState* old_jvms = call->jvms();
 733     JVMState* jvms = old_jvms->clone_shallow(C);
 734     uint size = call->req();
 735     SafePointNode* map = new SafePointNode(size, jvms);
 736     for (uint i1 = 0; i1 < size; i1++) {
 737       map->init_req(i1, call->in(i1));
 738     }
 739 
 740     PhaseGVN& gvn = *C->initial_gvn();
 741     // Make sure the state is a MergeMem for parsing.
 742     if (!map->in(TypeFunc::Memory)->is_MergeMem()) {
 743       Node* mem = MergeMemNode::make(map->in(TypeFunc::Memory));
 744       gvn.set_type_bottom(mem);
 745       map->set_req(TypeFunc::Memory, mem);
 746     }
 747 
 748     // blow away old call arguments
 749     for (uint i1 = TypeFunc::Parms; i1 < r->cnt(); i1++) {
 750       map->set_req(i1, C->top());
 751     }
 752     jvms->set_map(map);
 753 
 754     // Make enough space in the expression stack to transfer
 755     // the incoming arguments and return value.
 756     map->ensure_stack(jvms, jvms->method()->max_stack());
 757     const TypeTuple* domain_sig = call->_tf->domain_sig();
 758     uint nargs = method()->arg_size();
 759     assert(domain_sig->cnt() - TypeFunc::Parms == nargs, "inconsistent signature");
 760 
 761     uint j = TypeFunc::Parms;
 762     for (uint i1 = 0; i1 < nargs; i1++) {
 763       const Type* t = domain_sig->field_at(TypeFunc::Parms + i1);
 764       if (method()->has_scalarized_args() && t->is_inlinetypeptr() && !t->maybe_null() && t->inline_klass()->can_be_passed_as_fields()) {
 765         // Inline type arguments are not passed by reference: we get an argument per
 766         // field of the inline type. Build InlineTypeNodes from the inline type arguments.
 767         GraphKit arg_kit(jvms, &gvn);
 768         InlineTypeNode* vt = InlineTypeNode::make_from_multi(&arg_kit, call, t->inline_klass(), j, true);
 769         map->set_control(arg_kit.control());
 770         map->set_argument(jvms, i1, vt);
 771       } else {
 772         map->set_argument(jvms, i1, call->in(j++));
 773       }
 774     }
 775 
 776     C->print_inlining_assert_ready();
 777 
 778     C->print_inlining_move_to(this);
 779 
 780     C->log_late_inline(this);
 781 
 782     // JVMState is ready, so time to perform some checks and prepare for inlining attempt.
 783     if (!do_late_inline_check(C, jvms)) {
 784       map->disconnect_inputs(C);
 785       C->print_inlining_update_delayed(this);
 786       return;
 787     }
 788 
 789     // Check if we are late inlining a method handle call that returns an inline type as fields.
 790     Node* buffer_oop = NULL;
 791     ciType* mh_rt = inline_cg()->method()->return_type();
 792     if (is_mh_late_inline() && mh_rt->is_inlinetype() && mh_rt->as_inline_klass()->can_be_returned_as_fields()) {
 793       // Allocate a buffer for the inline type returned as fields because the caller expects an oop return.
 794       // Do this before the method handle call in case the buffer allocation triggers deoptimization and
 795       // we need to "re-execute" the call in the interpreter (to make sure the call is only executed once).
 796       GraphKit arg_kit(jvms, &gvn);
 797       {
 798         PreserveReexecuteState preexecs(&arg_kit);
 799         arg_kit.jvms()->set_should_reexecute(true);
 800         arg_kit.inc_sp(nargs);
 801         Node* klass_node = arg_kit.makecon(TypeKlassPtr::make(mh_rt->as_inline_klass()));
 802         buffer_oop = arg_kit.new_instance(klass_node, NULL, NULL, /* deoptimize_on_exception */ true);
 803       }
 804       jvms = arg_kit.transfer_exceptions_into_jvms();
 805     }
 806 
 807     // Setup default node notes to be picked up by the inlining
 808     Node_Notes* old_nn = C->node_notes_at(call->_idx);
 809     if (old_nn != NULL) {
 810       Node_Notes* entry_nn = old_nn->clone(C);
 811       entry_nn->set_jvms(jvms);
 812       C->set_default_node_notes(entry_nn);
 813     }
 814 
 815     // Now perform the inlining using the synthesized JVMState
 816     JVMState* new_jvms = inline_cg()->generate(jvms);
 817     if (new_jvms == NULL)  return;  // no change
 818     if (C->failing())      return;
 819 
 820     // Capture any exceptional control flow
 821     GraphKit kit(new_jvms);
 822 
 823     // Find the result object
 824     Node* result = C->top();
 825     int   result_size = method()->return_type()->size();
 826     if (result_size != 0 && !kit.stopped()) {
 827       result = (result_size == 1) ? kit.pop() : kit.pop_pair();
 828     }
 829 
 830     if (inline_cg()->is_inline()) {
 831       C->set_has_loops(C->has_loops() || inline_cg()->method()->has_loops());
 832       C->env()->notice_inlined_method(inline_cg()->method());
 833     }
 834     C->set_inlining_progress(true);
 835     C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup
 836 
 837     // Handle inline type returns
 838     InlineTypeNode* vt = result->isa_InlineType();
 839     if (vt != NULL) {
 840       if (call->tf()->returns_inline_type_as_fields()) {
 841         vt->replace_call_results(&kit, call, C);
 842       } else {
 843         // Result might still be allocated (for example, if it has been stored to a non-flattened field)
 844         if (!vt->is_allocated(&kit.gvn())) {
 845           assert(buffer_oop != NULL, "should have allocated a buffer");
 846           vt->store(&kit, buffer_oop, buffer_oop, vt->type()->inline_klass());
 847           // Do not let stores that initialize this buffer be reordered with a subsequent
 848           // store that would make this buffer accessible by other threads.
 849           AllocateNode* alloc = AllocateNode::Ideal_allocation(buffer_oop, &kit.gvn());
 850           assert(alloc != NULL, "must have an allocation node");
 851           kit.insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress));
 852           kit.gvn().hash_delete(vt);
 853           vt->set_oop(buffer_oop);
 854           vt = kit.gvn().transform(vt)->as_InlineType();
 855         }
 856         DEBUG_ONLY(buffer_oop = NULL);
 857         // Convert to InlineTypePtrNode to keep track of field values
 858         result = vt->as_ptr(&kit.gvn());
 859       }
 860     }
 861     assert(buffer_oop == NULL, "unused buffer allocation");
 862 
 863     kit.replace_call(call, result, true);
 864   }
 865 }
 866 
 867 class LateInlineStringCallGenerator : public LateInlineCallGenerator {
 868 
 869  public:
 870   LateInlineStringCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
 871     LateInlineCallGenerator(method, inline_cg) {}
 872 
 873   virtual JVMState* generate(JVMState* jvms) {
 874     Compile *C = Compile::current();
 875 
 876     C->log_inline_id(this);
 877 
 878     C->add_string_late_inline(this);
 879 
 880     JVMState* new_jvms = DirectCallGenerator::generate(jvms);
 881     return new_jvms;
 882   }
 883 
 884   virtual bool is_string_late_inline() const { return true; }
 885 
 886   virtual CallGenerator* with_call_node(CallNode* call) {
 887     LateInlineStringCallGenerator* cg = new LateInlineStringCallGenerator(method(), _inline_cg);
 888     cg->set_call_node(call->as_CallStaticJava());
 889     return cg;
 890   }
 891 };
 892 
 893 CallGenerator* CallGenerator::for_string_late_inline(ciMethod* method, CallGenerator* inline_cg) {
 894   return new LateInlineStringCallGenerator(method, inline_cg);
 895 }
 896 
 897 class LateInlineBoxingCallGenerator : public LateInlineCallGenerator {
 898 
 899  public:
 900   LateInlineBoxingCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
 901     LateInlineCallGenerator(method, inline_cg, /*is_pure=*/true) {}
 902 
 903   virtual JVMState* generate(JVMState* jvms) {
 904     Compile *C = Compile::current();
 905 
 906     C->log_inline_id(this);
 907 
 908     C->add_boxing_late_inline(this);
 909 
 910     JVMState* new_jvms = DirectCallGenerator::generate(jvms);
 911     return new_jvms;
 912   }
 913 
 914   virtual bool is_boxing_late_inline() const { return true; }
 915 
 916   virtual CallGenerator* with_call_node(CallNode* call) {
 917     LateInlineBoxingCallGenerator* cg = new LateInlineBoxingCallGenerator(method(), _inline_cg);
 918     cg->set_call_node(call->as_CallStaticJava());
 919     return cg;
 920   }
 921 };
 922 
 923 CallGenerator* CallGenerator::for_boxing_late_inline(ciMethod* method, CallGenerator* inline_cg) {
 924   return new LateInlineBoxingCallGenerator(method, inline_cg);
 925 }
 926 
 927 class LateInlineVectorReboxingCallGenerator : public LateInlineCallGenerator {
 928 
 929  public:
 930   LateInlineVectorReboxingCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
 931     LateInlineCallGenerator(method, inline_cg, /*is_pure=*/true) {}
 932 
 933   virtual JVMState* generate(JVMState* jvms) {
 934     Compile *C = Compile::current();
 935 
 936     C->log_inline_id(this);
 937 
 938     C->add_vector_reboxing_late_inline(this);
 939 
 940     JVMState* new_jvms = DirectCallGenerator::generate(jvms);
 941     return new_jvms;
 942   }
 943 
 944   virtual CallGenerator* with_call_node(CallNode* call) {
 945     LateInlineVectorReboxingCallGenerator* cg = new LateInlineVectorReboxingCallGenerator(method(), _inline_cg);
 946     cg->set_call_node(call->as_CallStaticJava());
 947     return cg;
 948   }
 949 };
 950 
 951 //   static CallGenerator* for_vector_reboxing_late_inline(ciMethod* m, CallGenerator* inline_cg);
 952 CallGenerator* CallGenerator::for_vector_reboxing_late_inline(ciMethod* method, CallGenerator* inline_cg) {
 953   return new LateInlineVectorReboxingCallGenerator(method, inline_cg);
 954 }
 955 
 956 //------------------------PredictedCallGenerator------------------------------
 957 // Internal class which handles all out-of-line calls checking receiver type.
 958 class PredictedCallGenerator : public CallGenerator {
 959   ciKlass*       _predicted_receiver;
 960   CallGenerator* _if_missed;
 961   CallGenerator* _if_hit;
 962   float          _hit_prob;
 963   bool           _exact_check;
 964 
 965 public:
 966   PredictedCallGenerator(ciKlass* predicted_receiver,
 967                          CallGenerator* if_missed,
 968                          CallGenerator* if_hit, bool exact_check,
 969                          float hit_prob)
 970     : CallGenerator(if_missed->method())
 971   {
 972     // The call profile data may predict the hit_prob as extreme as 0 or 1.
 973     // Remove the extremes values from the range.
 974     if (hit_prob > PROB_MAX)   hit_prob = PROB_MAX;
 975     if (hit_prob < PROB_MIN)   hit_prob = PROB_MIN;
 976 
 977     _predicted_receiver = predicted_receiver;
 978     _if_missed          = if_missed;
 979     _if_hit             = if_hit;
 980     _hit_prob           = hit_prob;
 981     _exact_check        = exact_check;
 982   }
 983 
 984   virtual bool      is_virtual()   const    { return true; }
 985   virtual bool      is_inline()    const    { return _if_hit->is_inline(); }
 986   virtual bool      is_deferred()  const    { return _if_hit->is_deferred(); }
 987 
 988   virtual JVMState* generate(JVMState* jvms);
 989 };
 990 
 991 
 992 CallGenerator* CallGenerator::for_predicted_call(ciKlass* predicted_receiver,
 993                                                  CallGenerator* if_missed,
 994                                                  CallGenerator* if_hit,
 995                                                  float hit_prob) {
 996   return new PredictedCallGenerator(predicted_receiver, if_missed, if_hit,
 997                                     /*exact_check=*/true, hit_prob);
 998 }
 999 
1000 CallGenerator* CallGenerator::for_guarded_call(ciKlass* guarded_receiver,
1001                                                CallGenerator* if_missed,
1002                                                CallGenerator* if_hit) {
1003   return new PredictedCallGenerator(guarded_receiver, if_missed, if_hit,
1004                                     /*exact_check=*/false, PROB_ALWAYS);
1005 }
1006 
1007 JVMState* PredictedCallGenerator::generate(JVMState* jvms) {
1008   GraphKit kit(jvms);
1009   kit.C->print_inlining_update(this);
1010   PhaseGVN& gvn = kit.gvn();
1011   // We need an explicit receiver null_check before checking its type.
1012   // We share a map with the caller, so his JVMS gets adjusted.
1013   Node* receiver = kit.argument(0);
1014   CompileLog* log = kit.C->log();
1015   if (log != NULL) {
1016     log->elem("predicted_call bci='%d' exact='%d' klass='%d'",
1017               jvms->bci(), (_exact_check ? 1 : 0), log->identify(_predicted_receiver));
1018   }
1019 
1020   receiver = kit.null_check_receiver_before_call(method());
1021   if (kit.stopped()) {
1022     return kit.transfer_exceptions_into_jvms();
1023   }
1024 
1025   // Make a copy of the replaced nodes in case we need to restore them
1026   ReplacedNodes replaced_nodes = kit.map()->replaced_nodes();
1027   replaced_nodes.clone();
1028 
1029   Node* casted_receiver = receiver;  // will get updated in place...
1030   Node* slow_ctl = NULL;
1031   if (_exact_check) {
1032     slow_ctl = kit.type_check_receiver(receiver, _predicted_receiver, _hit_prob,
1033                                        &casted_receiver);
1034   } else {
1035     slow_ctl = kit.subtype_check_receiver(receiver, _predicted_receiver,
1036                                           &casted_receiver);
1037   }
1038 
1039   SafePointNode* slow_map = NULL;
1040   JVMState* slow_jvms = NULL;
1041   { PreserveJVMState pjvms(&kit);
1042     kit.set_control(slow_ctl);
1043     if (!kit.stopped()) {
1044       slow_jvms = _if_missed->generate(kit.sync_jvms());
1045       if (kit.failing())
1046         return NULL;  // might happen because of NodeCountInliningCutoff
1047       assert(slow_jvms != NULL, "must be");
1048       kit.add_exception_states_from(slow_jvms);
1049       kit.set_map(slow_jvms->map());
1050       if (!kit.stopped())
1051         slow_map = kit.stop();
1052     }
1053   }
1054 
1055   if (kit.stopped()) {
1056     // Instance does not match the predicted type.
1057     kit.set_jvms(slow_jvms);
1058     return kit.transfer_exceptions_into_jvms();
1059   }
1060 
1061   // Fall through if the instance matches the desired type.
1062   kit.replace_in_map(receiver, casted_receiver);
1063 
1064   // Make the hot call:
1065   JVMState* new_jvms = _if_hit->generate(kit.sync_jvms());
1066   if (new_jvms == NULL) {
1067     // Inline failed, so make a direct call.
1068     assert(_if_hit->is_inline(), "must have been a failed inline");
1069     CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method());
1070     new_jvms = cg->generate(kit.sync_jvms());
1071   }
1072   kit.add_exception_states_from(new_jvms);
1073   kit.set_jvms(new_jvms);
1074 
1075   // Need to merge slow and fast?
1076   if (slow_map == NULL) {
1077     // The fast path is the only path remaining.
1078     return kit.transfer_exceptions_into_jvms();
1079   }
1080 
1081   if (kit.stopped()) {
1082     // Inlined method threw an exception, so it's just the slow path after all.
1083     kit.set_jvms(slow_jvms);
1084     return kit.transfer_exceptions_into_jvms();
1085   }
1086 
1087   // Allocate inline types if they are merged with objects (similar to Parse::merge_common())
1088   uint tos = kit.jvms()->stkoff() + kit.sp();
1089   uint limit = slow_map->req();
1090   for (uint i = TypeFunc::Parms; i < limit; i++) {
1091     Node* m = kit.map()->in(i);
1092     Node* n = slow_map->in(i);
1093     const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
1094     if (m->is_InlineType() && !t->isa_inlinetype()) {
1095       // Allocate inline type in fast path
1096       m = m->as_InlineType()->buffer(&kit);
1097       kit.map()->set_req(i, m);
1098     }
1099     if (n->is_InlineType() && !t->isa_inlinetype()) {
1100       // Allocate inline type in slow path
1101       PreserveJVMState pjvms(&kit);
1102       kit.set_map(slow_map);
1103       n = n->as_InlineType()->buffer(&kit);
1104       kit.map()->set_req(i, n);
1105       slow_map = kit.stop();
1106     }
1107   }
1108 
1109   // There are 2 branches and the replaced nodes are only valid on
1110   // one: restore the replaced nodes to what they were before the
1111   // branch.
1112   kit.map()->set_replaced_nodes(replaced_nodes);
1113 
1114   // Finish the diamond.
1115   kit.C->set_has_split_ifs(true); // Has chance for split-if optimization
1116   RegionNode* region = new RegionNode(3);
1117   region->init_req(1, kit.control());
1118   region->init_req(2, slow_map->control());
1119   kit.set_control(gvn.transform(region));
1120   Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO);
1121   iophi->set_req(2, slow_map->i_o());
1122   kit.set_i_o(gvn.transform(iophi));
1123   // Merge memory
1124   kit.merge_memory(slow_map->merged_memory(), region, 2);
1125   // Transform new memory Phis.
1126   for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) {
1127     Node* phi = mms.memory();
1128     if (phi->is_Phi() && phi->in(0) == region) {
1129       mms.set_memory(gvn.transform(phi));
1130     }
1131   }
1132   for (uint i = TypeFunc::Parms; i < limit; i++) {
1133     // Skip unused stack slots; fast forward to monoff();
1134     if (i == tos) {
1135       i = kit.jvms()->monoff();
1136       if( i >= limit ) break;
1137     }
1138     Node* m = kit.map()->in(i);
1139     Node* n = slow_map->in(i);
1140     if (m != n) {
1141       const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
1142       Node* phi = PhiNode::make(region, m, t);
1143       phi->set_req(2, n);
1144       kit.map()->set_req(i, gvn.transform(phi));
1145     }
1146   }
1147   return kit.transfer_exceptions_into_jvms();
1148 }
1149 
1150 
1151 CallGenerator* CallGenerator::for_method_handle_call(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline) {
1152   assert(callee->is_method_handle_intrinsic(), "for_method_handle_call mismatch");
1153   bool input_not_const;
1154   CallGenerator* cg = CallGenerator::for_method_handle_inline(jvms, caller, callee, allow_inline, input_not_const);
1155   Compile* C = Compile::current();
1156   if (cg != NULL) {
1157     if (AlwaysIncrementalInline) {
1158       return CallGenerator::for_late_inline(callee, cg);
1159     } else {
1160       return cg;
1161     }
1162   }
1163   int bci = jvms->bci();
1164   ciCallProfile profile = caller->call_profile_at_bci(bci);
1165   int call_site_count = caller->scale_count(profile.count());
1166 
1167   if (IncrementalInlineMH && (AlwaysIncrementalInline ||
1168                             (call_site_count > 0 && (input_not_const || !C->inlining_incrementally() || C->over_inlining_cutoff())))) {
1169     return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
1170   } else {
1171     // Out-of-line call.
1172     return CallGenerator::for_direct_call(callee);
1173   }
1174 }
1175 
1176 static void cast_argument(int nargs, int arg_nb, ciType* t, GraphKit& kit, bool null_free) {
1177   PhaseGVN& gvn = kit.gvn();
1178   Node* arg = kit.argument(arg_nb);
1179   const Type* arg_type = arg->bottom_type();
1180   const Type* sig_type = TypeOopPtr::make_from_klass(t->as_klass());
1181   if (t->as_klass()->is_inlinetype() && null_free) {
1182     sig_type = sig_type->filter_speculative(TypePtr::NOTNULL);
1183   }
1184   if (arg_type->isa_oopptr() && !arg_type->higher_equal(sig_type)) {
1185     const Type* narrowed_arg_type = arg_type->filter_speculative(sig_type); // keep speculative part
1186     arg = gvn.transform(new CheckCastPPNode(kit.control(), arg, narrowed_arg_type));
1187     kit.set_argument(arg_nb, arg);
1188   }
1189   if (sig_type->is_inlinetypeptr() && !arg->is_InlineType()) {
1190     arg = InlineTypeNode::make_from_oop(&kit, arg, t->as_inline_klass(), !kit.gvn().type(arg)->maybe_null());
1191     kit.set_argument(arg_nb, arg);
1192   }
1193 }
1194 
1195 class NativeCallGenerator : public CallGenerator {
1196 private:
1197   address _call_addr;
1198   ciNativeEntryPoint* _nep;
1199 public:
1200   NativeCallGenerator(ciMethod* m, address call_addr, ciNativeEntryPoint* nep)
1201    : CallGenerator(m), _call_addr(call_addr), _nep(nep) {}
1202 
1203   virtual JVMState* generate(JVMState* jvms);
1204 };
1205 
1206 JVMState* NativeCallGenerator::generate(JVMState* jvms) {
1207   GraphKit kit(jvms);
1208 
1209   Node* call = kit.make_native_call(_call_addr, tf(), method()->arg_size(), _nep); // -fallback, - nep
1210   if (call == NULL) return NULL;
1211 
1212   kit.C->print_inlining_update(this);
1213   if (kit.C->log() != NULL) {
1214     kit.C->log()->elem("l2n_intrinsification_success bci='%d' entry_point='" INTPTR_FORMAT "'", jvms->bci(), p2i(_call_addr));
1215   }
1216 
1217   return kit.transfer_exceptions_into_jvms();
1218 }
1219 
1220 CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline, bool& input_not_const) {
1221   GraphKit kit(jvms);
1222   PhaseGVN& gvn = kit.gvn();
1223   Compile* C = kit.C;
1224   vmIntrinsics::ID iid = callee->intrinsic_id();
1225   input_not_const = true;
1226   if (StressMethodHandleLinkerInlining) {
1227     allow_inline = false;
1228   }
1229   switch (iid) {
1230   case vmIntrinsics::_invokeBasic:
1231     {
1232       // Get MethodHandle receiver:
1233       Node* receiver = kit.argument(0);
1234       if (receiver->Opcode() == Op_ConP) {
1235         input_not_const = false;
1236         const TypeOopPtr* oop_ptr = receiver->bottom_type()->is_oopptr();
1237         ciMethod* target = oop_ptr->const_oop()->as_method_handle()->get_vmtarget();
1238         const int vtable_index = Method::invalid_vtable_index;
1239 
1240         if (!ciMethod::is_consistent_info(callee, target)) {
1241           print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1242                                  "signatures mismatch");
1243           return NULL;
1244         }
1245 
1246         CallGenerator* cg = C->call_generator(target, vtable_index,
1247                                               false /* call_does_dispatch */,
1248                                               jvms,
1249                                               allow_inline,
1250                                               PROB_ALWAYS,
1251                                               NULL,
1252                                               true);
1253         return cg;
1254       } else {
1255         print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1256                                "receiver not constant");
1257       }
1258     }
1259     break;
1260 
1261   case vmIntrinsics::_linkToVirtual:
1262   case vmIntrinsics::_linkToStatic:
1263   case vmIntrinsics::_linkToSpecial:
1264   case vmIntrinsics::_linkToInterface:
1265     {
1266       int nargs = callee->arg_size();
1267       // Get MemberName argument:
1268       Node* member_name = kit.argument(nargs - 1);
1269       if (member_name->Opcode() == Op_ConP) {
1270         input_not_const = false;
1271         const TypeOopPtr* oop_ptr = member_name->bottom_type()->is_oopptr();
1272         ciMethod* target = oop_ptr->const_oop()->as_member_name()->get_vmtarget();
1273 
1274         if (!ciMethod::is_consistent_info(callee, target)) {
1275           print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1276                                  "signatures mismatch");
1277           return NULL;
1278         }
1279 
1280         // In lambda forms we erase signature types to avoid resolving issues
1281         // involving class loaders.  When we optimize a method handle invoke
1282         // to a direct call we must cast the receiver and arguments to its
1283         // actual types.
1284         ciSignature* signature = target->signature();
1285         const int receiver_skip = target->is_static() ? 0 : 1;
1286         // Cast receiver to its type.
1287         if (!target->is_static()) {
1288           cast_argument(nargs, 0, signature->accessing_klass(), kit, false);
1289         }
1290         // Cast reference arguments to its type.
1291         for (int i = 0, j = 0; i < signature->count(); i++) {
1292           ciType* t = signature->type_at(i);
1293           if (t->is_klass()) {
1294             bool null_free = signature->is_null_free_at(i);
1295             cast_argument(nargs, receiver_skip + j, t, kit, null_free);
1296           }
1297           j += t->size();  // long and double take two slots
1298         }
1299 
1300         // Try to get the most accurate receiver type
1301         const bool is_virtual              = (iid == vmIntrinsics::_linkToVirtual);
1302         const bool is_virtual_or_interface = (is_virtual || iid == vmIntrinsics::_linkToInterface);
1303         int  vtable_index       = Method::invalid_vtable_index;
1304         bool call_does_dispatch = false;
1305 
1306         ciKlass* speculative_receiver_type = NULL;
1307         if (is_virtual_or_interface) {
1308           ciInstanceKlass* klass = target->holder();
1309           Node*             receiver_node = kit.argument(0);
1310           const TypeOopPtr* receiver_type = gvn.type(receiver_node)->isa_oopptr();
1311           // call_does_dispatch and vtable_index are out-parameters.  They might be changed.
1312           // optimize_virtual_call() takes 2 different holder
1313           // arguments for a corner case that doesn't apply here (see
1314           // Parse::do_call())
1315           target = C->optimize_virtual_call(caller, klass, klass,
1316                                             target, receiver_type, is_virtual,
1317                                             call_does_dispatch, vtable_index, // out-parameters
1318                                             false /* check_access */);
1319           // We lack profiling at this call but type speculation may
1320           // provide us with a type
1321           speculative_receiver_type = (receiver_type != NULL) ? receiver_type->speculative_type() : NULL;
1322         }
1323         CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms,
1324                                               allow_inline,
1325                                               PROB_ALWAYS,
1326                                               speculative_receiver_type,
1327                                               true);
1328         return cg;
1329       } else {
1330         print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1331                                "member_name not constant");
1332       }
1333     }
1334     break;
1335 
1336     case vmIntrinsics::_linkToNative:
1337     {
1338       Node* addr_n = kit.argument(1); // target address
1339       Node* nep_n = kit.argument(callee->arg_size() - 1); // NativeEntryPoint
1340       // This check needs to be kept in sync with the one in CallStaticJavaNode::Ideal
1341       if (addr_n->Opcode() == Op_ConL && nep_n->Opcode() == Op_ConP) {
1342         input_not_const = false;
1343         const TypeLong* addr_t = addr_n->bottom_type()->is_long();
1344         const TypeOopPtr* nep_t = nep_n->bottom_type()->is_oopptr();
1345         address addr = (address) addr_t->get_con();
1346         ciNativeEntryPoint* nep = nep_t->const_oop()->as_native_entry_point();
1347         return new NativeCallGenerator(callee, addr, nep);
1348       } else {
1349         print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1350                                "NativeEntryPoint not constant");
1351       }
1352     }
1353     break;
1354 
1355   default:
1356     fatal("unexpected intrinsic %d: %s", vmIntrinsics::as_int(iid), vmIntrinsics::name_at(iid));
1357     break;
1358   }
1359   return NULL;
1360 }
1361 
1362 
1363 //------------------------PredicatedIntrinsicGenerator------------------------------
1364 // Internal class which handles all predicated Intrinsic calls.
1365 class PredicatedIntrinsicGenerator : public CallGenerator {
1366   CallGenerator* _intrinsic;
1367   CallGenerator* _cg;
1368 
1369 public:
1370   PredicatedIntrinsicGenerator(CallGenerator* intrinsic,
1371                                CallGenerator* cg)
1372     : CallGenerator(cg->method())
1373   {
1374     _intrinsic = intrinsic;
1375     _cg        = cg;
1376   }
1377 
1378   virtual bool      is_virtual()   const    { return true; }
1379   virtual bool      is_inline()    const    { return true; }
1380   virtual bool      is_intrinsic() const    { return true; }
1381 
1382   virtual JVMState* generate(JVMState* jvms);
1383 };
1384 
1385 
1386 CallGenerator* CallGenerator::for_predicated_intrinsic(CallGenerator* intrinsic,
1387                                                        CallGenerator* cg) {
1388   return new PredicatedIntrinsicGenerator(intrinsic, cg);
1389 }
1390 
1391 
1392 JVMState* PredicatedIntrinsicGenerator::generate(JVMState* jvms) {
1393   // The code we want to generate here is:
1394   //    if (receiver == NULL)
1395   //        uncommon_Trap
1396   //    if (predicate(0))
1397   //        do_intrinsic(0)
1398   //    else
1399   //    if (predicate(1))
1400   //        do_intrinsic(1)
1401   //    ...
1402   //    else
1403   //        do_java_comp
1404 
1405   GraphKit kit(jvms);
1406   PhaseGVN& gvn = kit.gvn();
1407 
1408   CompileLog* log = kit.C->log();
1409   if (log != NULL) {
1410     log->elem("predicated_intrinsic bci='%d' method='%d'",
1411               jvms->bci(), log->identify(method()));
1412   }
1413 
1414   if (!method()->is_static()) {
1415     // We need an explicit receiver null_check before checking its type in predicate.
1416     // We share a map with the caller, so his JVMS gets adjusted.
1417     kit.null_check_receiver_before_call(method());
1418     if (kit.stopped()) {
1419       return kit.transfer_exceptions_into_jvms();
1420     }
1421   }
1422 
1423   int n_predicates = _intrinsic->predicates_count();
1424   assert(n_predicates > 0, "sanity");
1425 
1426   JVMState** result_jvms = NEW_RESOURCE_ARRAY(JVMState*, (n_predicates+1));
1427 
1428   // Region for normal compilation code if intrinsic failed.
1429   Node* slow_region = new RegionNode(1);
1430 
1431   int results = 0;
1432   for (int predicate = 0; (predicate < n_predicates) && !kit.stopped(); predicate++) {
1433 #ifdef ASSERT
1434     JVMState* old_jvms = kit.jvms();
1435     SafePointNode* old_map = kit.map();
1436     Node* old_io  = old_map->i_o();
1437     Node* old_mem = old_map->memory();
1438     Node* old_exc = old_map->next_exception();
1439 #endif
1440     Node* else_ctrl = _intrinsic->generate_predicate(kit.sync_jvms(), predicate);
1441 #ifdef ASSERT
1442     // Assert(no_new_memory && no_new_io && no_new_exceptions) after generate_predicate.
1443     assert(old_jvms == kit.jvms(), "generate_predicate should not change jvm state");
1444     SafePointNode* new_map = kit.map();
1445     assert(old_io  == new_map->i_o(), "generate_predicate should not change i_o");
1446     assert(old_mem == new_map->memory(), "generate_predicate should not change memory");
1447     assert(old_exc == new_map->next_exception(), "generate_predicate should not add exceptions");
1448 #endif
1449     if (!kit.stopped()) {
1450       PreserveJVMState pjvms(&kit);
1451       // Generate intrinsic code:
1452       JVMState* new_jvms = _intrinsic->generate(kit.sync_jvms());
1453       if (new_jvms == NULL) {
1454         // Intrinsic failed, use normal compilation path for this predicate.
1455         slow_region->add_req(kit.control());
1456       } else {
1457         kit.add_exception_states_from(new_jvms);
1458         kit.set_jvms(new_jvms);
1459         if (!kit.stopped()) {
1460           result_jvms[results++] = kit.jvms();
1461         }
1462       }
1463     }
1464     if (else_ctrl == NULL) {
1465       else_ctrl = kit.C->top();
1466     }
1467     kit.set_control(else_ctrl);
1468   }
1469   if (!kit.stopped()) {
1470     // Final 'else' after predicates.
1471     slow_region->add_req(kit.control());
1472   }
1473   if (slow_region->req() > 1) {
1474     PreserveJVMState pjvms(&kit);
1475     // Generate normal compilation code:
1476     kit.set_control(gvn.transform(slow_region));
1477     JVMState* new_jvms = _cg->generate(kit.sync_jvms());
1478     if (kit.failing())
1479       return NULL;  // might happen because of NodeCountInliningCutoff
1480     assert(new_jvms != NULL, "must be");
1481     kit.add_exception_states_from(new_jvms);
1482     kit.set_jvms(new_jvms);
1483     if (!kit.stopped()) {
1484       result_jvms[results++] = kit.jvms();
1485     }
1486   }
1487 
1488   if (results == 0) {
1489     // All paths ended in uncommon traps.
1490     (void) kit.stop();
1491     return kit.transfer_exceptions_into_jvms();
1492   }
1493 
1494   if (results == 1) { // Only one path
1495     kit.set_jvms(result_jvms[0]);
1496     return kit.transfer_exceptions_into_jvms();
1497   }
1498 
1499   // Merge all paths.
1500   kit.C->set_has_split_ifs(true); // Has chance for split-if optimization
1501   RegionNode* region = new RegionNode(results + 1);
1502   Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO);
1503   for (int i = 0; i < results; i++) {
1504     JVMState* jvms = result_jvms[i];
1505     int path = i + 1;
1506     SafePointNode* map = jvms->map();
1507     region->init_req(path, map->control());
1508     iophi->set_req(path, map->i_o());
1509     if (i == 0) {
1510       kit.set_jvms(jvms);
1511     } else {
1512       kit.merge_memory(map->merged_memory(), region, path);
1513     }
1514   }
1515   kit.set_control(gvn.transform(region));
1516   kit.set_i_o(gvn.transform(iophi));
1517   // Transform new memory Phis.
1518   for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) {
1519     Node* phi = mms.memory();
1520     if (phi->is_Phi() && phi->in(0) == region) {
1521       mms.set_memory(gvn.transform(phi));
1522     }
1523   }
1524 
1525   // Merge debug info.
1526   Node** ins = NEW_RESOURCE_ARRAY(Node*, results);
1527   uint tos = kit.jvms()->stkoff() + kit.sp();
1528   Node* map = kit.map();
1529   uint limit = map->req();
1530   for (uint i = TypeFunc::Parms; i < limit; i++) {
1531     // Skip unused stack slots; fast forward to monoff();
1532     if (i == tos) {
1533       i = kit.jvms()->monoff();
1534       if( i >= limit ) break;
1535     }
1536     Node* n = map->in(i);
1537     ins[0] = n;
1538     const Type* t = gvn.type(n);
1539     bool needs_phi = false;
1540     for (int j = 1; j < results; j++) {
1541       JVMState* jvms = result_jvms[j];
1542       Node* jmap = jvms->map();
1543       Node* m = NULL;
1544       if (jmap->req() > i) {
1545         m = jmap->in(i);
1546         if (m != n) {
1547           needs_phi = true;
1548           t = t->meet_speculative(gvn.type(m));
1549         }
1550       }
1551       ins[j] = m;
1552     }
1553     if (needs_phi) {
1554       Node* phi = PhiNode::make(region, n, t);
1555       for (int j = 1; j < results; j++) {
1556         phi->set_req(j + 1, ins[j]);
1557       }
1558       map->set_req(i, gvn.transform(phi));
1559     }
1560   }
1561 
1562   return kit.transfer_exceptions_into_jvms();
1563 }
1564 
1565 //-------------------------UncommonTrapCallGenerator-----------------------------
1566 // Internal class which handles all out-of-line calls checking receiver type.
1567 class UncommonTrapCallGenerator : public CallGenerator {
1568   Deoptimization::DeoptReason _reason;
1569   Deoptimization::DeoptAction _action;
1570 
1571 public:
1572   UncommonTrapCallGenerator(ciMethod* m,
1573                             Deoptimization::DeoptReason reason,
1574                             Deoptimization::DeoptAction action)
1575     : CallGenerator(m)
1576   {
1577     _reason = reason;
1578     _action = action;
1579   }
1580 
1581   virtual bool      is_virtual() const          { ShouldNotReachHere(); return false; }
1582   virtual bool      is_trap() const             { return true; }
1583 
1584   virtual JVMState* generate(JVMState* jvms);
1585 };
1586 
1587 
1588 CallGenerator*
1589 CallGenerator::for_uncommon_trap(ciMethod* m,
1590                                  Deoptimization::DeoptReason reason,
1591                                  Deoptimization::DeoptAction action) {
1592   return new UncommonTrapCallGenerator(m, reason, action);
1593 }
1594 
1595 
1596 JVMState* UncommonTrapCallGenerator::generate(JVMState* jvms) {
1597   GraphKit kit(jvms);
1598   kit.C->print_inlining_update(this);
1599   // Take the trap with arguments pushed on the stack.  (Cf. null_check_receiver).
1600   // Callsite signature can be different from actual method being called (i.e _linkTo* sites).
1601   // Use callsite signature always.
1602   ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci());
1603   int nargs = declared_method->arg_size();
1604   kit.inc_sp(nargs);
1605   assert(nargs <= kit.sp() && kit.sp() <= jvms->stk_size(), "sane sp w/ args pushed");
1606   if (_reason == Deoptimization::Reason_class_check &&
1607       _action == Deoptimization::Action_maybe_recompile) {
1608     // Temp fix for 6529811
1609     // Don't allow uncommon_trap to override our decision to recompile in the event
1610     // of a class cast failure for a monomorphic call as it will never let us convert
1611     // the call to either bi-morphic or megamorphic and can lead to unc-trap loops
1612     bool keep_exact_action = true;
1613     kit.uncommon_trap(_reason, _action, NULL, "monomorphic vcall checkcast", false, keep_exact_action);
1614   } else {
1615     kit.uncommon_trap(_reason, _action);
1616   }
1617   return kit.transfer_exceptions_into_jvms();
1618 }
1619 
1620 // (Note:  Moved hook_up_call to GraphKit::set_edges_for_java_call.)
1621 
1622 // (Node:  Merged hook_up_exits into ParseGenerator::generate.)