< prev index next >

src/hotspot/share/opto/callGenerator.cpp

Print this page

  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "ci/bcEscapeAnalyzer.hpp"
  26 #include "ci/ciCallSite.hpp"
  27 #include "ci/ciObjArray.hpp"
  28 #include "ci/ciMemberName.hpp"
  29 #include "ci/ciMethodHandle.hpp"
  30 #include "classfile/javaClasses.hpp"
  31 #include "compiler/compileLog.hpp"

  32 #include "opto/addnode.hpp"
  33 #include "opto/callGenerator.hpp"
  34 #include "opto/callnode.hpp"
  35 #include "opto/castnode.hpp"
  36 #include "opto/cfgnode.hpp"

  37 #include "opto/parse.hpp"
  38 #include "opto/rootnode.hpp"
  39 #include "opto/runtime.hpp"
  40 #include "opto/subnode.hpp"
  41 #include "runtime/os.inline.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "utilities/debug.hpp"
  44 
  45 // Utility function.
  46 const TypeFunc* CallGenerator::tf() const {
  47   return TypeFunc::make(method());
  48 }
  49 
  50 bool CallGenerator::is_inlined_method_handle_intrinsic(JVMState* jvms, ciMethod* m) {
  51   return is_inlined_method_handle_intrinsic(jvms->method(), jvms->bci(), m);
  52 }
  53 
  54 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* caller, int bci, ciMethod* m) {
  55   ciMethod* symbolic_info = caller->get_method_at_bci(bci);
  56   return is_inlined_method_handle_intrinsic(symbolic_info, m);

 101   GraphKit& exits = parser.exits();
 102 
 103   if (C->failing()) {
 104     while (exits.pop_exception_state() != nullptr) ;
 105     return nullptr;
 106   }
 107 
 108   assert(exits.jvms()->same_calls_as(jvms), "sanity");
 109 
 110   // Simply return the exit state of the parser,
 111   // augmented by any exceptional states.
 112   return exits.transfer_exceptions_into_jvms();
 113 }
 114 
 115 //---------------------------DirectCallGenerator------------------------------
 116 // Internal class which handles all out-of-line calls w/o receiver type checks.
 117 class DirectCallGenerator : public CallGenerator {
 118  private:
 119   CallStaticJavaNode* _call_node;
 120   // Force separate memory and I/O projections for the exceptional
 121   // paths to facilitate late inlinig.
 122   bool                _separate_io_proj;
 123 
 124 protected:
 125   void set_call_node(CallStaticJavaNode* call) { _call_node = call; }
 126 
 127  public:
 128   DirectCallGenerator(ciMethod* method, bool separate_io_proj)
 129     : CallGenerator(method),

 130       _separate_io_proj(separate_io_proj)
 131   {








 132   }
 133   virtual JVMState* generate(JVMState* jvms);
 134 
 135   virtual CallNode* call_node() const { return _call_node; }
 136   virtual CallGenerator* with_call_node(CallNode* call) {
 137     DirectCallGenerator* dcg = new DirectCallGenerator(method(), _separate_io_proj);
 138     dcg->set_call_node(call->as_CallStaticJava());
 139     return dcg;
 140   }
 141 };
 142 
 143 JVMState* DirectCallGenerator::generate(JVMState* jvms) {
 144   GraphKit kit(jvms);

 145   bool is_static = method()->is_static();
 146   address target = is_static ? SharedRuntime::get_resolve_static_call_stub()
 147                              : SharedRuntime::get_resolve_opt_virtual_call_stub();
 148 
 149   if (kit.C->log() != nullptr) {
 150     kit.C->log()->elem("direct_call bci='%d'", jvms->bci());
 151   }
 152 
 153   CallStaticJavaNode* call = new CallStaticJavaNode(kit.C, tf(), target, method());
 154   if (is_inlined_method_handle_intrinsic(jvms, method())) {
 155     // To be able to issue a direct call and skip a call to MH.linkTo*/invokeBasic adapter,
 156     // additional information about the method being invoked should be attached
 157     // to the call site to make resolution logic work
 158     // (see SharedRuntime::resolve_static_call_C).
 159     call->set_override_symbolic_info(true);
 160   }
 161   _call_node = call;  // Save the call node in case we need it later
 162   if (!is_static) {
 163     // Make an explicit receiver null_check as part of this call.
 164     // Since we share a map with the caller, his JVMS gets adjusted.
 165     kit.null_check_receiver_before_call(method());
 166     if (kit.stopped()) {
 167       // And dump it back to the caller, decorated with any exceptions:
 168       return kit.transfer_exceptions_into_jvms();
 169     }
 170     // Mark the call node as virtual, sort of:
 171     call->set_optimized_virtual(true);
 172     if (method()->is_method_handle_intrinsic() ||
 173         method()->is_compiled_lambda_form()) {
 174       call->set_method_handle_invoke(true);
 175     }
 176   }
 177   kit.set_arguments_for_java_call(call);



 178   kit.set_edges_for_java_call(call, false, _separate_io_proj);
 179   Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
 180   kit.push_node(method()->return_type()->basic_type(), ret);
 181   return kit.transfer_exceptions_into_jvms();
 182 }
 183 
 184 //--------------------------VirtualCallGenerator------------------------------
 185 // Internal class which handles all out-of-line calls checking receiver type.
 186 class VirtualCallGenerator : public CallGenerator {
 187 private:
 188   int _vtable_index;
 189   bool _separate_io_proj;
 190   CallDynamicJavaNode* _call_node;
 191 
 192 protected:
 193   void set_call_node(CallDynamicJavaNode* call) { _call_node = call; }
 194 
 195 public:
 196   VirtualCallGenerator(ciMethod* method, int vtable_index, bool separate_io_proj)
 197     : CallGenerator(method), _vtable_index(vtable_index), _separate_io_proj(separate_io_proj), _call_node(nullptr)
 198   {
 199     assert(vtable_index == Method::invalid_vtable_index ||
 200            vtable_index >= 0, "either invalid or usable");
 201   }
 202   virtual bool      is_virtual() const          { return true; }
 203   virtual JVMState* generate(JVMState* jvms);
 204 
 205   virtual CallNode* call_node() const { return _call_node; }
 206   int vtable_index() const { return _vtable_index; }
 207 
 208   virtual CallGenerator* with_call_node(CallNode* call) {
 209     VirtualCallGenerator* cg = new VirtualCallGenerator(method(), _vtable_index, _separate_io_proj);
 210     cg->set_call_node(call->as_CallDynamicJava());
 211     return cg;
 212   }
 213 };
 214 
 215 JVMState* VirtualCallGenerator::generate(JVMState* jvms) {
 216   GraphKit kit(jvms);
 217   Node* receiver = kit.argument(0);
 218 
 219   if (kit.C->log() != nullptr) {
 220     kit.C->log()->elem("virtual_call bci='%d'", jvms->bci());
 221   }
 222 
 223   // If the receiver is a constant null, do not torture the system
 224   // by attempting to call through it.  The compile will proceed
 225   // correctly, but may bail out in final_graph_reshaping, because
 226   // the call instruction will have a seemingly deficient out-count.
 227   // (The bailout says something misleading about an "infinite loop".)
 228   if (kit.gvn().type(receiver)->higher_equal(TypePtr::NULL_PTR)) {
 229     assert(Bytecodes::is_invoke(kit.java_bc()), "%d: %s", kit.java_bc(), Bytecodes::name(kit.java_bc()));
 230     ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci());
 231     int arg_size = declared_method->signature()->arg_size_for_bc(kit.java_bc());
 232     kit.inc_sp(arg_size);  // restore arguments
 233     kit.uncommon_trap(Deoptimization::Reason_null_check,
 234                       Deoptimization::Action_none,
 235                       nullptr, "null receiver");
 236     return kit.transfer_exceptions_into_jvms();
 237   }
 238 

 256   }
 257 
 258   assert(!method()->is_static(), "virtual call must not be to static");
 259   assert(!method()->is_final(), "virtual call should not be to final");
 260   assert(!method()->is_private(), "virtual call should not be to private");
 261   assert(_vtable_index == Method::invalid_vtable_index || !UseInlineCaches,
 262          "no vtable calls if +UseInlineCaches ");
 263   address target = SharedRuntime::get_resolve_virtual_call_stub();
 264   // Normal inline cache used for call
 265   CallDynamicJavaNode* call = new CallDynamicJavaNode(tf(), target, method(), _vtable_index);
 266   if (is_inlined_method_handle_intrinsic(jvms, method())) {
 267     // To be able to issue a direct call (optimized virtual or virtual)
 268     // and skip a call to MH.linkTo*/invokeBasic adapter, additional information
 269     // about the method being invoked should be attached to the call site to
 270     // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C).
 271     call->set_override_symbolic_info(true);
 272   }
 273   _call_node = call;  // Save the call node in case we need it later
 274 
 275   kit.set_arguments_for_java_call(call);



 276   kit.set_edges_for_java_call(call, false /*must_throw*/, _separate_io_proj);
 277   Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
 278   kit.push_node(method()->return_type()->basic_type(), ret);
 279 
 280   // Represent the effect of an implicit receiver null_check
 281   // as part of this call.  Since we share a map with the caller,
 282   // his JVMS gets adjusted.
 283   kit.cast_not_null(receiver);
 284   return kit.transfer_exceptions_into_jvms();
 285 }
 286 
 287 CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) {
 288   if (InlineTree::check_can_parse(m) != nullptr)  return nullptr;
 289   return new ParseGenerator(m, expected_uses);
 290 }
 291 
 292 // As a special case, the JVMS passed to this CallGenerator is
 293 // for the method execution already in progress, not just the JVMS
 294 // of the caller.  Thus, this CallGenerator cannot be mixed with others!
 295 CallGenerator* CallGenerator::for_osr(ciMethod* m, int osr_bci) {

 340     // parse is finished.
 341     if (!is_mh_late_inline()) {
 342       C->add_late_inline(this);
 343     }
 344 
 345     // Emit the CallStaticJava and request separate projections so
 346     // that the late inlining logic can distinguish between fall
 347     // through and exceptional uses of the memory and io projections
 348     // as is done for allocations and macro expansion.
 349     return DirectCallGenerator::generate(jvms);
 350   }
 351 
 352   virtual void set_unique_id(jlong id) {
 353     _unique_id = id;
 354   }
 355 
 356   virtual jlong unique_id() const {
 357     return _unique_id;
 358   }
 359 




 360   virtual CallGenerator* with_call_node(CallNode* call) {
 361     LateInlineCallGenerator* cg = new LateInlineCallGenerator(method(), _inline_cg, _is_pure_call);
 362     cg->set_call_node(call->as_CallStaticJava());
 363     return cg;
 364   }
 365 };
 366 
 367 CallGenerator* CallGenerator::for_late_inline(ciMethod* method, CallGenerator* inline_cg) {
 368   return new LateInlineCallGenerator(method, inline_cg);
 369 }
 370 
 371 class LateInlineMHCallGenerator : public LateInlineCallGenerator {
 372   ciMethod* _caller;
 373   bool _input_not_const;
 374 
 375   virtual bool do_late_inline_check(Compile* C, JVMState* jvms);
 376 
 377  public:
 378   LateInlineMHCallGenerator(ciMethod* caller, ciMethod* callee, bool input_not_const) :
 379     LateInlineCallGenerator(callee, nullptr), _caller(caller), _input_not_const(input_not_const) {}

 401     cg->set_call_node(call->as_CallStaticJava());
 402     return cg;
 403   }
 404 };
 405 
 406 bool LateInlineMHCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) {
 407   // When inlining a virtual call, the null check at the call and the call itself can throw. These 2 paths have different
 408   // expression stacks which causes late inlining to break. The MH invoker is not expected to be called from a method with
 409   // exception handlers. When there is no exception handler, GraphKit::builtin_throw() pops the stack which solves the issue
 410   // of late inlining with exceptions.
 411   assert(!jvms->method()->has_exception_handlers() ||
 412          (method()->intrinsic_id() != vmIntrinsics::_linkToVirtual &&
 413           method()->intrinsic_id() != vmIntrinsics::_linkToInterface), "no exception handler expected");
 414   // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call.
 415   bool allow_inline = C->inlining_incrementally();
 416   bool input_not_const = true;
 417   CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), allow_inline, input_not_const);
 418   assert(!input_not_const, "sanity"); // shouldn't have been scheduled for inlining in the first place
 419 
 420   if (cg != nullptr) {








 421     if (!allow_inline) {
 422       C->inline_printer()->record(cg->method(), call_node()->jvms(), InliningResult::FAILURE,
 423                                   "late method handle call resolution");
 424     }
 425     assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline || StressIncrementalInlining, "we're doing late inlining");
 426     _inline_cg = cg;
 427     C->dec_number_of_mh_late_inlines();
 428     return true;
 429   } else {
 430     // Method handle call which has a constant appendix argument should be either inlined or replaced with a direct call
 431     // unless there's a signature mismatch between caller and callee. If the failure occurs, there's not much to be improved later,
 432     // so don't reinstall the generator to avoid pushing the generator between IGVN and incremental inlining indefinitely.
 433     return false;
 434   }
 435 }
 436 
 437 CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const) {
 438   assert(IncrementalInlineMH, "required");
 439   Compile::current()->inc_number_of_mh_late_inlines();
 440   CallGenerator* cg = new LateInlineMHCallGenerator(caller, callee, input_not_const);

 557 
 558 void LateInlineMHCallGenerator::do_late_inline() {
 559   CallGenerator::do_late_inline_helper();
 560 }
 561 
 562 void LateInlineVirtualCallGenerator::do_late_inline() {
 563   assert(_callee != nullptr, "required"); // set up in CallDynamicJavaNode::Ideal
 564   CallGenerator::do_late_inline_helper();
 565 }
 566 
 567 void CallGenerator::do_late_inline_helper() {
 568   assert(is_late_inline(), "only late inline allowed");
 569 
 570   // Can't inline it
 571   CallNode* call = call_node();
 572   if (call == nullptr || call->outcnt() == 0 ||
 573       call->in(0) == nullptr || call->in(0)->is_top()) {
 574     return;
 575   }
 576 
 577   const TypeTuple *r = call->tf()->domain();
 578   for (int i1 = 0; i1 < method()->arg_size(); i1++) {
 579     if (call->in(TypeFunc::Parms + i1)->is_top() && r->field_at(TypeFunc::Parms + i1) != Type::HALF) {
 580       assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
 581       return;
 582     }
 583   }
 584 
 585   if (call->in(TypeFunc::Memory)->is_top()) {
 586     assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
 587     return;
 588   }
 589   if (call->in(TypeFunc::Memory)->is_MergeMem()) {
 590     MergeMemNode* merge_mem = call->in(TypeFunc::Memory)->as_MergeMem();
 591     if (merge_mem->base_memory() == merge_mem->empty_memory()) {
 592       return; // dead path
 593     }
 594   }
 595 
 596   // check for unreachable loop
 597   CallProjections callprojs;
 598   // Similar to incremental inlining, don't assert that all call
 599   // projections are still there for post-parse call devirtualization.
 600   bool do_asserts = !is_mh_late_inline() && !is_virtual_late_inline();
 601   call->extract_projections(&callprojs, true, do_asserts);
 602   if ((callprojs.fallthrough_catchproj == call->in(0)) ||
 603       (callprojs.catchall_catchproj    == call->in(0)) ||
 604       (callprojs.fallthrough_memproj   == call->in(TypeFunc::Memory)) ||
 605       (callprojs.catchall_memproj      == call->in(TypeFunc::Memory)) ||
 606       (callprojs.fallthrough_ioproj    == call->in(TypeFunc::I_O)) ||
 607       (callprojs.catchall_ioproj       == call->in(TypeFunc::I_O)) ||
 608       (callprojs.resproj != nullptr && call->find_edge(callprojs.resproj) != -1) ||
 609       (callprojs.exobj   != nullptr && call->find_edge(callprojs.exobj) != -1)) {
 610     return;
 611   }
 612 
 613   Compile* C = Compile::current();
 614   // Remove inlined methods from Compiler's lists.
 615   if (call->is_macro()) {
 616     C->remove_macro_node(call);
 617   }
 618 
 619   // The call is marked as pure (no important side effects), but result isn't used.
 620   // It's safe to remove the call.
 621   bool result_not_used = (callprojs.resproj == nullptr || callprojs.resproj->outcnt() == 0);









 622 
 623   if (is_pure_call() && result_not_used) {


 624     GraphKit kit(call->jvms());
 625     kit.replace_call(call, C->top(), true, do_asserts);
 626   } else {
 627     // Make a clone of the JVMState that appropriate to use for driving a parse
 628     JVMState* old_jvms = call->jvms();
 629     JVMState* jvms = old_jvms->clone_shallow(C);
 630     uint size = call->req();
 631     SafePointNode* map = new SafePointNode(size, jvms);
 632     for (uint i1 = 0; i1 < size; i1++) {
 633       map->init_req(i1, call->in(i1));
 634     }
 635 

 636     // Make sure the state is a MergeMem for parsing.
 637     if (!map->in(TypeFunc::Memory)->is_MergeMem()) {
 638       Node* mem = MergeMemNode::make(map->in(TypeFunc::Memory));
 639       C->initial_gvn()->set_type_bottom(mem);
 640       map->set_req(TypeFunc::Memory, mem);
 641     }
 642 
 643     uint nargs = method()->arg_size();
 644     // blow away old call arguments
 645     Node* top = C->top();
 646     for (uint i1 = 0; i1 < nargs; i1++) {
 647       map->set_req(TypeFunc::Parms + i1, top);
 648     }
 649     jvms->set_map(map);
 650 
 651     // Make enough space in the expression stack to transfer
 652     // the incoming arguments and return value.
 653     map->ensure_stack(jvms, jvms->method()->max_stack());






 654     for (uint i1 = 0; i1 < nargs; i1++) {
 655       map->set_argument(jvms, i1, call->in(TypeFunc::Parms + i1));













 656     }
 657 
 658     C->log_late_inline(this);
 659 
 660     // JVMState is ready, so time to perform some checks and prepare for inlining attempt.
 661     if (!do_late_inline_check(C, jvms)) {
 662       map->disconnect_inputs(C);
 663       return;
 664     }
 665 




















 666     // Setup default node notes to be picked up by the inlining
 667     Node_Notes* old_nn = C->node_notes_at(call->_idx);
 668     if (old_nn != nullptr) {
 669       Node_Notes* entry_nn = old_nn->clone(C);
 670       entry_nn->set_jvms(jvms);
 671       C->set_default_node_notes(entry_nn);
 672     }
 673 
 674     // Now perform the inlining using the synthesized JVMState
 675     JVMState* new_jvms = inline_cg()->generate(jvms);
 676     if (new_jvms == nullptr)  return;  // no change
 677     if (C->failing())      return;
 678 
 679     if (is_mh_late_inline()) {
 680       C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (method handle)");
 681     } else if (is_string_late_inline()) {
 682       C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (string method)");
 683     } else if (is_boxing_late_inline()) {
 684       C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (boxing method)");
 685     } else if (is_vector_reboxing_late_inline()) {
 686       C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (vector reboxing method)");
 687     } else {
 688       C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded");
 689     }
 690 
 691     // Capture any exceptional control flow
 692     GraphKit kit(new_jvms);
 693 
 694     // Find the result object
 695     Node* result = C->top();
 696     int   result_size = method()->return_type()->size();
 697     if (result_size != 0 && !kit.stopped()) {
 698       result = (result_size == 1) ? kit.pop() : kit.pop_pair();
 699     }
 700 
 701     if (call->is_CallStaticJava() && call->as_CallStaticJava()->is_boxing_method()) {
 702       result = kit.must_be_not_null(result, false);
 703     }
 704 
 705     if (inline_cg()->is_inline()) {
 706       C->set_has_loops(C->has_loops() || inline_cg()->method()->has_loops());
 707       C->env()->notice_inlined_method(inline_cg()->method());
 708     }
 709     C->set_inlining_progress(true);
 710     C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup






















































 711     kit.replace_call(call, result, true, do_asserts);
 712   }
 713 }
 714 
 715 class LateInlineStringCallGenerator : public LateInlineCallGenerator {
 716 
 717  public:
 718   LateInlineStringCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
 719     LateInlineCallGenerator(method, inline_cg) {}
 720 
 721   virtual JVMState* generate(JVMState* jvms) {
 722     Compile *C = Compile::current();
 723 
 724     C->log_inline_id(this);
 725 
 726     C->add_string_late_inline(this);
 727 
 728     JVMState* new_jvms = DirectCallGenerator::generate(jvms);
 729     return new_jvms;
 730   }

 919     // Inline failed, so make a direct call.
 920     assert(_if_hit->is_inline(), "must have been a failed inline");
 921     CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method());
 922     new_jvms = cg->generate(kit.sync_jvms());
 923   }
 924   kit.add_exception_states_from(new_jvms);
 925   kit.set_jvms(new_jvms);
 926 
 927   // Need to merge slow and fast?
 928   if (slow_map == nullptr) {
 929     // The fast path is the only path remaining.
 930     return kit.transfer_exceptions_into_jvms();
 931   }
 932 
 933   if (kit.stopped()) {
 934     // Inlined method threw an exception, so it's just the slow path after all.
 935     kit.set_jvms(slow_jvms);
 936     return kit.transfer_exceptions_into_jvms();
 937   }
 938 























 939   // There are 2 branches and the replaced nodes are only valid on
 940   // one: restore the replaced nodes to what they were before the
 941   // branch.
 942   kit.map()->set_replaced_nodes(replaced_nodes);
 943 
 944   // Finish the diamond.
 945   kit.C->set_has_split_ifs(true); // Has chance for split-if optimization
 946   RegionNode* region = new RegionNode(3);
 947   region->init_req(1, kit.control());
 948   region->init_req(2, slow_map->control());
 949   kit.set_control(gvn.transform(region));
 950   Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO);
 951   iophi->set_req(2, slow_map->i_o());
 952   kit.set_i_o(gvn.transform(iophi));
 953   // Merge memory
 954   kit.merge_memory(slow_map->merged_memory(), region, 2);
 955   // Transform new memory Phis.
 956   for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) {
 957     Node* phi = mms.memory();
 958     if (phi->is_Phi() && phi->in(0) == region) {
 959       mms.set_memory(gvn.transform(phi));
 960     }
 961   }
 962   uint tos = kit.jvms()->stkoff() + kit.sp();
 963   uint limit = slow_map->req();
 964   for (uint i = TypeFunc::Parms; i < limit; i++) {
 965     // Skip unused stack slots; fast forward to monoff();
 966     if (i == tos) {
 967       i = kit.jvms()->monoff();
 968       if( i >= limit ) break;
 969     }
 970     Node* m = kit.map()->in(i);
 971     Node* n = slow_map->in(i);
 972     if (m != n) {
 973       const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
 974       Node* phi = PhiNode::make(region, m, t);
 975       phi->set_req(2, n);
 976       kit.map()->set_req(i, gvn.transform(phi));
 977     }
 978   }
 979   return kit.transfer_exceptions_into_jvms();
 980 }
 981 
 982 
 983 CallGenerator* CallGenerator::for_method_handle_call(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline) {
 984   assert(callee->is_method_handle_intrinsic(), "for_method_handle_call mismatch");
 985   bool input_not_const;
 986   CallGenerator* cg = CallGenerator::for_method_handle_inline(jvms, caller, callee, allow_inline, input_not_const);
 987   Compile* C = Compile::current();
 988   bool should_delay = C->should_delay_inlining();
 989   if (cg != nullptr) {
 990     if (should_delay && IncrementalInlineMH) {
 991       return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
 992     } else {
 993       return cg;
 994     }
 995   }
 996   int bci = jvms->bci();
 997   ciCallProfile profile = caller->call_profile_at_bci(bci);
 998   int call_site_count = caller->scale_count(profile.count());
 999 
1000   if (IncrementalInlineMH && call_site_count > 0 &&
1001       (should_delay || input_not_const || !C->inlining_incrementally() || C->over_inlining_cutoff())) {
1002     return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
1003   } else {
1004     // Out-of-line call.
1005     return CallGenerator::for_direct_call(callee);
1006   }
1007 }
1008 

1009 CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline, bool& input_not_const) {
1010   GraphKit kit(jvms);
1011   PhaseGVN& gvn = kit.gvn();
1012   Compile* C = kit.C;
1013   vmIntrinsics::ID iid = callee->intrinsic_id();
1014   input_not_const = true;
1015   if (StressMethodHandleLinkerInlining) {
1016     allow_inline = false;
1017   }
1018   switch (iid) {
1019   case vmIntrinsics::_invokeBasic:
1020     {
1021       // Get MethodHandle receiver:
1022       Node* receiver = kit.argument(0);
1023       if (receiver->Opcode() == Op_ConP) {
1024         input_not_const = false;
1025         const TypeOopPtr* recv_toop = receiver->bottom_type()->isa_oopptr();
1026         if (recv_toop != nullptr) {
1027           ciMethod* target = recv_toop->const_oop()->as_method_handle()->get_vmtarget();
1028           const int vtable_index = Method::invalid_vtable_index;

1036                                                 false /* call_does_dispatch */,
1037                                                 jvms,
1038                                                 allow_inline,
1039                                                 PROB_ALWAYS);
1040           return cg;
1041         } else {
1042           assert(receiver->bottom_type() == TypePtr::NULL_PTR, "not a null: %s",
1043                  Type::str(receiver->bottom_type()));
1044           print_inlining_failure(C, callee, jvms, "receiver is always null");
1045         }
1046       } else {
1047         print_inlining_failure(C, callee, jvms, "receiver not constant");
1048       }
1049   } break;
1050 
1051   case vmIntrinsics::_linkToVirtual:
1052   case vmIntrinsics::_linkToStatic:
1053   case vmIntrinsics::_linkToSpecial:
1054   case vmIntrinsics::_linkToInterface:
1055     {

1056       // Get MemberName argument:
1057       Node* member_name = kit.argument(callee->arg_size() - 1);
1058       if (member_name->Opcode() == Op_ConP) {
1059         input_not_const = false;
1060         const TypeOopPtr* oop_ptr = member_name->bottom_type()->is_oopptr();
1061         ciMethod* target = oop_ptr->const_oop()->as_member_name()->get_vmtarget();
1062 
1063         if (!ciMethod::is_consistent_info(callee, target)) {
1064           print_inlining_failure(C, callee, jvms, "signatures mismatch");
1065           return nullptr;
1066         }
1067 
1068         // In lambda forms we erase signature types to avoid resolving issues
1069         // involving class loaders.  When we optimize a method handle invoke
1070         // to a direct call we must cast the receiver and arguments to its
1071         // actual types.
1072         ciSignature* signature = target->signature();
1073         const int receiver_skip = target->is_static() ? 0 : 1;
1074         // Cast receiver to its type.
1075         if (!target->is_static()) {
1076           Node* recv = kit.argument(0);
1077           Node* casted_recv = kit.maybe_narrow_object_type(recv, signature->accessing_klass());

1107         ciKlass* speculative_receiver_type = nullptr;
1108         if (is_virtual_or_interface) {
1109           ciInstanceKlass* klass = target->holder();
1110           Node*             receiver_node = kit.argument(0);
1111           const TypeOopPtr* receiver_type = gvn.type(receiver_node)->isa_oopptr();
1112           // call_does_dispatch and vtable_index are out-parameters.  They might be changed.
1113           // optimize_virtual_call() takes 2 different holder
1114           // arguments for a corner case that doesn't apply here (see
1115           // Parse::do_call())
1116           target = C->optimize_virtual_call(caller, klass, klass,
1117                                             target, receiver_type, is_virtual,
1118                                             call_does_dispatch, vtable_index, // out-parameters
1119                                             false /* check_access */);
1120           // We lack profiling at this call but type speculation may
1121           // provide us with a type
1122           speculative_receiver_type = (receiver_type != nullptr) ? receiver_type->speculative_type() : nullptr;
1123         }
1124         CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms,
1125                                               allow_inline,
1126                                               PROB_ALWAYS,
1127                                               speculative_receiver_type);

1128         return cg;
1129       } else {
1130         print_inlining_failure(C, callee, jvms, "member_name not constant");
1131       }
1132   } break;
1133 
1134   case vmIntrinsics::_linkToNative:
1135     print_inlining_failure(C, callee, jvms, "native call");
1136     break;
1137 
1138   default:
1139     fatal("unexpected intrinsic %d: %s", vmIntrinsics::as_int(iid), vmIntrinsics::name_at(iid));
1140     break;
1141   }
1142   return nullptr;
1143 }
1144 
1145 //------------------------PredicatedIntrinsicGenerator------------------------------
1146 // Internal class which handles all predicated Intrinsic calls.
1147 class PredicatedIntrinsicGenerator : public CallGenerator {

1179   //        do_intrinsic(0)
1180   //    else
1181   //    if (predicate(1))
1182   //        do_intrinsic(1)
1183   //    ...
1184   //    else
1185   //        do_java_comp
1186 
1187   GraphKit kit(jvms);
1188   PhaseGVN& gvn = kit.gvn();
1189 
1190   CompileLog* log = kit.C->log();
1191   if (log != nullptr) {
1192     log->elem("predicated_intrinsic bci='%d' method='%d'",
1193               jvms->bci(), log->identify(method()));
1194   }
1195 
1196   if (!method()->is_static()) {
1197     // We need an explicit receiver null_check before checking its type in predicate.
1198     // We share a map with the caller, so his JVMS gets adjusted.
1199     Node* receiver = kit.null_check_receiver_before_call(method());
1200     if (kit.stopped()) {
1201       return kit.transfer_exceptions_into_jvms();
1202     }
1203   }
1204 
1205   int n_predicates = _intrinsic->predicates_count();
1206   assert(n_predicates > 0, "sanity");
1207 
1208   JVMState** result_jvms = NEW_RESOURCE_ARRAY(JVMState*, (n_predicates+1));
1209 
1210   // Region for normal compilation code if intrinsic failed.
1211   Node* slow_region = new RegionNode(1);
1212 
1213   int results = 0;
1214   for (int predicate = 0; (predicate < n_predicates) && !kit.stopped(); predicate++) {
1215 #ifdef ASSERT
1216     JVMState* old_jvms = kit.jvms();
1217     SafePointNode* old_map = kit.map();
1218     Node* old_io  = old_map->i_o();
1219     Node* old_mem = old_map->memory();

  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "ci/bcEscapeAnalyzer.hpp"
  26 #include "ci/ciCallSite.hpp"
  27 #include "ci/ciObjArray.hpp"
  28 #include "ci/ciMemberName.hpp"
  29 #include "ci/ciMethodHandle.hpp"
  30 #include "classfile/javaClasses.hpp"
  31 #include "compiler/compileLog.hpp"
  32 #include "oops/accessDecorators.hpp"
  33 #include "opto/addnode.hpp"
  34 #include "opto/callGenerator.hpp"
  35 #include "opto/callnode.hpp"
  36 #include "opto/castnode.hpp"
  37 #include "opto/cfgnode.hpp"
  38 #include "opto/inlinetypenode.hpp"
  39 #include "opto/parse.hpp"
  40 #include "opto/rootnode.hpp"
  41 #include "opto/runtime.hpp"
  42 #include "opto/subnode.hpp"
  43 #include "runtime/os.inline.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "utilities/debug.hpp"
  46 
  47 // Utility function.
  48 const TypeFunc* CallGenerator::tf() const {
  49   return TypeFunc::make(method());
  50 }
  51 
  52 bool CallGenerator::is_inlined_method_handle_intrinsic(JVMState* jvms, ciMethod* m) {
  53   return is_inlined_method_handle_intrinsic(jvms->method(), jvms->bci(), m);
  54 }
  55 
  56 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* caller, int bci, ciMethod* m) {
  57   ciMethod* symbolic_info = caller->get_method_at_bci(bci);
  58   return is_inlined_method_handle_intrinsic(symbolic_info, m);

 103   GraphKit& exits = parser.exits();
 104 
 105   if (C->failing()) {
 106     while (exits.pop_exception_state() != nullptr) ;
 107     return nullptr;
 108   }
 109 
 110   assert(exits.jvms()->same_calls_as(jvms), "sanity");
 111 
 112   // Simply return the exit state of the parser,
 113   // augmented by any exceptional states.
 114   return exits.transfer_exceptions_into_jvms();
 115 }
 116 
 117 //---------------------------DirectCallGenerator------------------------------
 118 // Internal class which handles all out-of-line calls w/o receiver type checks.
 119 class DirectCallGenerator : public CallGenerator {
 120  private:
 121   CallStaticJavaNode* _call_node;
 122   // Force separate memory and I/O projections for the exceptional
 123   // paths to facilitate late inlining.
 124   bool                _separate_io_proj;
 125 
 126 protected:
 127   void set_call_node(CallStaticJavaNode* call) { _call_node = call; }
 128 
 129  public:
 130   DirectCallGenerator(ciMethod* method, bool separate_io_proj)
 131     : CallGenerator(method),
 132       _call_node(nullptr),
 133       _separate_io_proj(separate_io_proj)
 134   {
 135     if (InlineTypeReturnedAsFields && method->is_method_handle_intrinsic()) {
 136       // If that call has not been optimized by the time optimizations are over,
 137       // we'll need to add a call to create an inline type instance from the klass
 138       // returned by the call (see PhaseMacroExpand::expand_mh_intrinsic_return).
 139       // Separating memory and I/O projections for exceptions is required to
 140       // perform that graph transformation.
 141       _separate_io_proj = true;
 142     }
 143   }
 144   virtual JVMState* generate(JVMState* jvms);
 145 
 146   virtual CallNode* call_node() const { return _call_node; }
 147   virtual CallGenerator* with_call_node(CallNode* call) {
 148     DirectCallGenerator* dcg = new DirectCallGenerator(method(), _separate_io_proj);
 149     dcg->set_call_node(call->as_CallStaticJava());
 150     return dcg;
 151   }
 152 };
 153 
 154 JVMState* DirectCallGenerator::generate(JVMState* jvms) {
 155   GraphKit kit(jvms);
 156   PhaseGVN& gvn = kit.gvn();
 157   bool is_static = method()->is_static();
 158   address target = is_static ? SharedRuntime::get_resolve_static_call_stub()
 159                              : SharedRuntime::get_resolve_opt_virtual_call_stub();
 160 
 161   if (kit.C->log() != nullptr) {
 162     kit.C->log()->elem("direct_call bci='%d'", jvms->bci());
 163   }
 164 
 165   CallStaticJavaNode* call = new CallStaticJavaNode(kit.C, tf(), target, method());
 166   if (is_inlined_method_handle_intrinsic(jvms, method())) {
 167     // To be able to issue a direct call and skip a call to MH.linkTo*/invokeBasic adapter,
 168     // additional information about the method being invoked should be attached
 169     // to the call site to make resolution logic work
 170     // (see SharedRuntime::resolve_static_call_C).
 171     call->set_override_symbolic_info(true);
 172   }
 173   _call_node = call;  // Save the call node in case we need it later
 174   if (!is_static) {
 175     // Make an explicit receiver null_check as part of this call.
 176     // Since we share a map with the caller, his JVMS gets adjusted.
 177     kit.null_check_receiver_before_call(method());
 178     if (kit.stopped()) {
 179       // And dump it back to the caller, decorated with any exceptions:
 180       return kit.transfer_exceptions_into_jvms();
 181     }
 182     // Mark the call node as virtual, sort of:
 183     call->set_optimized_virtual(true);
 184     if (method()->is_method_handle_intrinsic() ||
 185         method()->is_compiled_lambda_form()) {
 186       call->set_method_handle_invoke(true);
 187     }
 188   }
 189   kit.set_arguments_for_java_call(call, is_late_inline());
 190   if (kit.stopped()) {
 191     return kit.transfer_exceptions_into_jvms();
 192   }
 193   kit.set_edges_for_java_call(call, false, _separate_io_proj);
 194   Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
 195   kit.push_node(method()->return_type()->basic_type(), ret);
 196   return kit.transfer_exceptions_into_jvms();
 197 }
 198 
 199 //--------------------------VirtualCallGenerator------------------------------
 200 // Internal class which handles all out-of-line calls checking receiver type.
 201 class VirtualCallGenerator : public CallGenerator {
 202 private:
 203   int _vtable_index;
 204   bool _separate_io_proj;
 205   CallDynamicJavaNode* _call_node;
 206 
 207 protected:
 208   void set_call_node(CallDynamicJavaNode* call) { _call_node = call; }
 209 
 210 public:
 211   VirtualCallGenerator(ciMethod* method, int vtable_index, bool separate_io_proj)
 212     : CallGenerator(method), _vtable_index(vtable_index), _separate_io_proj(separate_io_proj), _call_node(nullptr)
 213   {
 214     assert(vtable_index == Method::invalid_vtable_index ||
 215            vtable_index >= 0, "either invalid or usable");
 216   }
 217   virtual bool      is_virtual() const          { return true; }
 218   virtual JVMState* generate(JVMState* jvms);
 219 
 220   virtual CallNode* call_node() const { return _call_node; }
 221   int vtable_index() const { return _vtable_index; }
 222 
 223   virtual CallGenerator* with_call_node(CallNode* call) {
 224     VirtualCallGenerator* cg = new VirtualCallGenerator(method(), _vtable_index, _separate_io_proj);
 225     cg->set_call_node(call->as_CallDynamicJava());
 226     return cg;
 227   }
 228 };
 229 
 230 JVMState* VirtualCallGenerator::generate(JVMState* jvms) {
 231   GraphKit kit(jvms);
 232   Node* receiver = kit.argument(0);

 233   if (kit.C->log() != nullptr) {
 234     kit.C->log()->elem("virtual_call bci='%d'", jvms->bci());
 235   }
 236 
 237   // If the receiver is a constant null, do not torture the system
 238   // by attempting to call through it.  The compile will proceed
 239   // correctly, but may bail out in final_graph_reshaping, because
 240   // the call instruction will have a seemingly deficient out-count.
 241   // (The bailout says something misleading about an "infinite loop".)
 242   if (kit.gvn().type(receiver)->higher_equal(TypePtr::NULL_PTR)) {
 243     assert(Bytecodes::is_invoke(kit.java_bc()), "%d: %s", kit.java_bc(), Bytecodes::name(kit.java_bc()));
 244     ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci());
 245     int arg_size = declared_method->signature()->arg_size_for_bc(kit.java_bc());
 246     kit.inc_sp(arg_size);  // restore arguments
 247     kit.uncommon_trap(Deoptimization::Reason_null_check,
 248                       Deoptimization::Action_none,
 249                       nullptr, "null receiver");
 250     return kit.transfer_exceptions_into_jvms();
 251   }
 252 

 270   }
 271 
 272   assert(!method()->is_static(), "virtual call must not be to static");
 273   assert(!method()->is_final(), "virtual call should not be to final");
 274   assert(!method()->is_private(), "virtual call should not be to private");
 275   assert(_vtable_index == Method::invalid_vtable_index || !UseInlineCaches,
 276          "no vtable calls if +UseInlineCaches ");
 277   address target = SharedRuntime::get_resolve_virtual_call_stub();
 278   // Normal inline cache used for call
 279   CallDynamicJavaNode* call = new CallDynamicJavaNode(tf(), target, method(), _vtable_index);
 280   if (is_inlined_method_handle_intrinsic(jvms, method())) {
 281     // To be able to issue a direct call (optimized virtual or virtual)
 282     // and skip a call to MH.linkTo*/invokeBasic adapter, additional information
 283     // about the method being invoked should be attached to the call site to
 284     // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C).
 285     call->set_override_symbolic_info(true);
 286   }
 287   _call_node = call;  // Save the call node in case we need it later
 288 
 289   kit.set_arguments_for_java_call(call);
 290   if (kit.stopped()) {
 291     return kit.transfer_exceptions_into_jvms();
 292   }
 293   kit.set_edges_for_java_call(call, false /*must_throw*/, _separate_io_proj);
 294   Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
 295   kit.push_node(method()->return_type()->basic_type(), ret);
 296 
 297   // Represent the effect of an implicit receiver null_check
 298   // as part of this call.  Since we share a map with the caller,
 299   // his JVMS gets adjusted.
 300   kit.cast_not_null(receiver);
 301   return kit.transfer_exceptions_into_jvms();
 302 }
 303 
 304 CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) {
 305   if (InlineTree::check_can_parse(m) != nullptr)  return nullptr;
 306   return new ParseGenerator(m, expected_uses);
 307 }
 308 
 309 // As a special case, the JVMS passed to this CallGenerator is
 310 // for the method execution already in progress, not just the JVMS
 311 // of the caller.  Thus, this CallGenerator cannot be mixed with others!
 312 CallGenerator* CallGenerator::for_osr(ciMethod* m, int osr_bci) {

 357     // parse is finished.
 358     if (!is_mh_late_inline()) {
 359       C->add_late_inline(this);
 360     }
 361 
 362     // Emit the CallStaticJava and request separate projections so
 363     // that the late inlining logic can distinguish between fall
 364     // through and exceptional uses of the memory and io projections
 365     // as is done for allocations and macro expansion.
 366     return DirectCallGenerator::generate(jvms);
 367   }
 368 
 369   virtual void set_unique_id(jlong id) {
 370     _unique_id = id;
 371   }
 372 
 373   virtual jlong unique_id() const {
 374     return _unique_id;
 375   }
 376 
 377   virtual CallGenerator* inline_cg() {
 378     return _inline_cg;
 379   }
 380 
 381   virtual CallGenerator* with_call_node(CallNode* call) {
 382     LateInlineCallGenerator* cg = new LateInlineCallGenerator(method(), _inline_cg, _is_pure_call);
 383     cg->set_call_node(call->as_CallStaticJava());
 384     return cg;
 385   }
 386 };
 387 
 388 CallGenerator* CallGenerator::for_late_inline(ciMethod* method, CallGenerator* inline_cg) {
 389   return new LateInlineCallGenerator(method, inline_cg);
 390 }
 391 
 392 class LateInlineMHCallGenerator : public LateInlineCallGenerator {
 393   ciMethod* _caller;
 394   bool _input_not_const;
 395 
 396   virtual bool do_late_inline_check(Compile* C, JVMState* jvms);
 397 
 398  public:
 399   LateInlineMHCallGenerator(ciMethod* caller, ciMethod* callee, bool input_not_const) :
 400     LateInlineCallGenerator(callee, nullptr), _caller(caller), _input_not_const(input_not_const) {}

 422     cg->set_call_node(call->as_CallStaticJava());
 423     return cg;
 424   }
 425 };
 426 
 427 bool LateInlineMHCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) {
 428   // When inlining a virtual call, the null check at the call and the call itself can throw. These 2 paths have different
 429   // expression stacks which causes late inlining to break. The MH invoker is not expected to be called from a method with
 430   // exception handlers. When there is no exception handler, GraphKit::builtin_throw() pops the stack which solves the issue
 431   // of late inlining with exceptions.
 432   assert(!jvms->method()->has_exception_handlers() ||
 433          (method()->intrinsic_id() != vmIntrinsics::_linkToVirtual &&
 434           method()->intrinsic_id() != vmIntrinsics::_linkToInterface), "no exception handler expected");
 435   // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call.
 436   bool allow_inline = C->inlining_incrementally();
 437   bool input_not_const = true;
 438   CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), allow_inline, input_not_const);
 439   assert(!input_not_const, "sanity"); // shouldn't have been scheduled for inlining in the first place
 440 
 441   if (cg != nullptr) {
 442     // AlwaysIncrementalInline causes for_method_handle_inline() to
 443     // return a LateInlineCallGenerator. Extract the
 444     // InlineCallGenerator from it.
 445     if (AlwaysIncrementalInline && cg->is_late_inline() && !cg->is_virtual_late_inline()) {
 446       cg = cg->inline_cg();
 447       assert(cg != nullptr, "inline call generator expected");
 448     }
 449 
 450     if (!allow_inline) {
 451       C->inline_printer()->record(cg->method(), call_node()->jvms(), InliningResult::FAILURE,
 452                                   "late method handle call resolution");
 453     }
 454     assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline || StressIncrementalInlining, "we're doing late inlining");
 455     _inline_cg = cg;
 456     C->dec_number_of_mh_late_inlines();
 457     return true;
 458   } else {
 459     // Method handle call which has a constant appendix argument should be either inlined or replaced with a direct call
 460     // unless there's a signature mismatch between caller and callee. If the failure occurs, there's not much to be improved later,
 461     // so don't reinstall the generator to avoid pushing the generator between IGVN and incremental inlining indefinitely.
 462     return false;
 463   }
 464 }
 465 
 466 CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const) {
 467   assert(IncrementalInlineMH, "required");
 468   Compile::current()->inc_number_of_mh_late_inlines();
 469   CallGenerator* cg = new LateInlineMHCallGenerator(caller, callee, input_not_const);

 586 
 587 void LateInlineMHCallGenerator::do_late_inline() {
 588   CallGenerator::do_late_inline_helper();
 589 }
 590 
 591 void LateInlineVirtualCallGenerator::do_late_inline() {
 592   assert(_callee != nullptr, "required"); // set up in CallDynamicJavaNode::Ideal
 593   CallGenerator::do_late_inline_helper();
 594 }
 595 
 596 void CallGenerator::do_late_inline_helper() {
 597   assert(is_late_inline(), "only late inline allowed");
 598 
 599   // Can't inline it
 600   CallNode* call = call_node();
 601   if (call == nullptr || call->outcnt() == 0 ||
 602       call->in(0) == nullptr || call->in(0)->is_top()) {
 603     return;
 604   }
 605 
 606   const TypeTuple* r = call->tf()->domain_cc();
 607   for (uint i1 = TypeFunc::Parms; i1 < r->cnt(); i1++) {
 608     if (call->in(i1)->is_top() && r->field_at(i1) != Type::HALF) {
 609       assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
 610       return;
 611     }
 612   }
 613 
 614   if (call->in(TypeFunc::Memory)->is_top()) {
 615     assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
 616     return;
 617   }
 618   if (call->in(TypeFunc::Memory)->is_MergeMem()) {
 619     MergeMemNode* merge_mem = call->in(TypeFunc::Memory)->as_MergeMem();
 620     if (merge_mem->base_memory() == merge_mem->empty_memory()) {
 621       return; // dead path
 622     }
 623   }
 624 
 625   // check for unreachable loop

 626   // Similar to incremental inlining, don't assert that all call
 627   // projections are still there for post-parse call devirtualization.
 628   bool do_asserts = !is_mh_late_inline() && !is_virtual_late_inline();
 629   CallProjections* callprojs = call->extract_projections(true, do_asserts);
 630   if ((callprojs->fallthrough_catchproj == call->in(0)) ||
 631       (callprojs->catchall_catchproj    == call->in(0)) ||
 632       (callprojs->fallthrough_memproj   == call->in(TypeFunc::Memory)) ||
 633       (callprojs->catchall_memproj      == call->in(TypeFunc::Memory)) ||
 634       (callprojs->fallthrough_ioproj    == call->in(TypeFunc::I_O)) ||
 635       (callprojs->catchall_ioproj       == call->in(TypeFunc::I_O)) ||
 636       (callprojs->exobj != nullptr && call->find_edge(callprojs->exobj) != -1)) {

 637     return;
 638   }
 639 
 640   Compile* C = Compile::current();
 641   // Remove inlined methods from Compiler's lists.
 642   if (call->is_macro()) {
 643     C->remove_macro_node(call);
 644   }
 645 
 646 
 647   bool result_not_used = true;
 648   for (uint i = 0; i < callprojs->nb_resproj; i++) {
 649     if (callprojs->resproj[i] != nullptr) {
 650       if (callprojs->resproj[i]->outcnt() != 0) {
 651         result_not_used = false;
 652       }
 653       if (call->find_edge(callprojs->resproj[i]) != -1) {
 654         return;
 655       }
 656     }
 657   }
 658 
 659   if (is_pure_call() && result_not_used) {
 660     // The call is marked as pure (no important side effects), but result isn't used.
 661     // It's safe to remove the call.
 662     GraphKit kit(call->jvms());
 663     kit.replace_call(call, C->top(), true, do_asserts);
 664   } else {
 665     // Make a clone of the JVMState that appropriate to use for driving a parse
 666     JVMState* old_jvms = call->jvms();
 667     JVMState* jvms = old_jvms->clone_shallow(C);
 668     uint size = call->req();
 669     SafePointNode* map = new SafePointNode(size, jvms);
 670     for (uint i1 = 0; i1 < size; i1++) {
 671       map->init_req(i1, call->in(i1));
 672     }
 673 
 674     PhaseGVN& gvn = *C->initial_gvn();
 675     // Make sure the state is a MergeMem for parsing.
 676     if (!map->in(TypeFunc::Memory)->is_MergeMem()) {
 677       Node* mem = MergeMemNode::make(map->in(TypeFunc::Memory));
 678       gvn.set_type_bottom(mem);
 679       map->set_req(TypeFunc::Memory, mem);
 680     }
 681 

 682     // blow away old call arguments
 683     for (uint i1 = TypeFunc::Parms; i1 < r->cnt(); i1++) {
 684       map->set_req(i1, C->top());

 685     }
 686     jvms->set_map(map);
 687 
 688     // Make enough space in the expression stack to transfer
 689     // the incoming arguments and return value.
 690     map->ensure_stack(jvms, jvms->method()->max_stack());
 691     const TypeTuple* domain_sig = call->_tf->domain_sig();
 692     uint nargs = method()->arg_size();
 693     assert(domain_sig->cnt() - TypeFunc::Parms == nargs, "inconsistent signature");
 694 
 695     uint j = TypeFunc::Parms;
 696     int arg_num = 0;
 697     for (uint i1 = 0; i1 < nargs; i1++) {
 698       const Type* t = domain_sig->field_at(TypeFunc::Parms + i1);
 699       if (t->is_inlinetypeptr() && !method()->get_Method()->mismatch() && method()->is_scalarized_arg(arg_num)) {
 700         // Inline type arguments are not passed by reference: we get an argument per
 701         // field of the inline type. Build InlineTypeNodes from the inline type arguments.
 702         GraphKit arg_kit(jvms, &gvn);
 703         Node* vt = InlineTypeNode::make_from_multi(&arg_kit, call, t->inline_klass(), j, /* in= */ true, /* null_free= */ !t->maybe_null());
 704         map->set_control(arg_kit.control());
 705         map->set_argument(jvms, i1, vt);
 706       } else {
 707         map->set_argument(jvms, i1, call->in(j++));
 708       }
 709       if (t != Type::HALF) {
 710         arg_num++;
 711       }
 712     }
 713 
 714     C->log_late_inline(this);
 715 
 716     // JVMState is ready, so time to perform some checks and prepare for inlining attempt.
 717     if (!do_late_inline_check(C, jvms)) {
 718       map->disconnect_inputs(C);
 719       return;
 720     }
 721 
 722     // Check if we are late inlining a method handle call that returns an inline type as fields.
 723     Node* buffer_oop = nullptr;
 724     ciMethod* inline_method = inline_cg()->method();
 725     ciType* return_type = inline_method->return_type();
 726     if (!call->tf()->returns_inline_type_as_fields() &&
 727         return_type->is_inlinetype() && return_type->as_inline_klass()->can_be_returned_as_fields()) {
 728       // Allocate a buffer for the inline type returned as fields because the caller expects an oop return.
 729       // Do this before the method handle call in case the buffer allocation triggers deoptimization and
 730       // we need to "re-execute" the call in the interpreter (to make sure the call is only executed once).
 731       GraphKit arg_kit(jvms, &gvn);
 732       {
 733         PreserveReexecuteState preexecs(&arg_kit);
 734         arg_kit.jvms()->set_should_reexecute(true);
 735         arg_kit.inc_sp(nargs);
 736         Node* klass_node = arg_kit.makecon(TypeKlassPtr::make(return_type->as_inline_klass()));
 737         buffer_oop = arg_kit.new_instance(klass_node, nullptr, nullptr, /* deoptimize_on_exception */ true);
 738       }
 739       jvms = arg_kit.transfer_exceptions_into_jvms();
 740     }
 741 
 742     // Setup default node notes to be picked up by the inlining
 743     Node_Notes* old_nn = C->node_notes_at(call->_idx);
 744     if (old_nn != nullptr) {
 745       Node_Notes* entry_nn = old_nn->clone(C);
 746       entry_nn->set_jvms(jvms);
 747       C->set_default_node_notes(entry_nn);
 748     }
 749 
 750     // Now perform the inlining using the synthesized JVMState
 751     JVMState* new_jvms = inline_cg()->generate(jvms);
 752     if (new_jvms == nullptr)  return;  // no change
 753     if (C->failing())      return;
 754 
 755     if (is_mh_late_inline()) {
 756       C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (method handle)");
 757     } else if (is_string_late_inline()) {
 758       C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (string method)");
 759     } else if (is_boxing_late_inline()) {
 760       C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (boxing method)");
 761     } else if (is_vector_reboxing_late_inline()) {
 762       C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (vector reboxing method)");
 763     } else {
 764       C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded");
 765     }
 766 
 767     // Capture any exceptional control flow
 768     GraphKit kit(new_jvms);
 769 
 770     // Find the result object
 771     Node* result = C->top();
 772     int   result_size = method()->return_type()->size();
 773     if (result_size != 0 && !kit.stopped()) {
 774       result = (result_size == 1) ? kit.pop() : kit.pop_pair();
 775     }
 776 
 777     if (call->is_CallStaticJava() && call->as_CallStaticJava()->is_boxing_method()) {
 778       result = kit.must_be_not_null(result, false);
 779     }
 780 
 781     if (inline_cg()->is_inline()) {
 782       C->set_has_loops(C->has_loops() || inline_method->has_loops());
 783       C->env()->notice_inlined_method(inline_method);
 784     }
 785     C->set_inlining_progress(true);
 786     C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup
 787 
 788     // Handle inline type returns
 789     InlineTypeNode* vt = result->isa_InlineType();
 790     if (vt != nullptr) {
 791       if (call->tf()->returns_inline_type_as_fields()) {
 792         vt->replace_call_results(&kit, call, C);
 793       } else {
 794         // Result might still be allocated (for example, if it has been stored to a non-flat field)
 795         if (!vt->is_allocated(&kit.gvn())) {
 796           assert(buffer_oop != nullptr, "should have allocated a buffer");
 797           RegionNode* region = new RegionNode(3);
 798 
 799           // Check if result is null
 800           Node* null_ctl = kit.top();
 801           kit.null_check_common(vt->get_is_init(), T_INT, false, &null_ctl);
 802           region->init_req(1, null_ctl);
 803           PhiNode* oop = PhiNode::make(region, kit.gvn().zerocon(T_OBJECT), TypeInstPtr::make(TypePtr::BotPTR, vt->type()->inline_klass()));
 804           Node* init_mem = kit.reset_memory();
 805           PhiNode* mem = PhiNode::make(region, init_mem, Type::MEMORY, TypePtr::BOTTOM);
 806 
 807           // Not null, initialize the buffer
 808           kit.set_all_memory(init_mem);
 809 
 810           Node* payload_ptr = kit.basic_plus_adr(buffer_oop, kit.gvn().type(vt)->inline_klass()->payload_offset());
 811           vt->store_flat(&kit, buffer_oop, payload_ptr, false, true, true, IN_HEAP | MO_UNORDERED);
 812           // Do not let stores that initialize this buffer be reordered with a subsequent
 813           // store that would make this buffer accessible by other threads.
 814           AllocateNode* alloc = AllocateNode::Ideal_allocation(buffer_oop);
 815           assert(alloc != nullptr, "must have an allocation node");
 816           kit.insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress));
 817           region->init_req(2, kit.control());
 818           oop->init_req(2, buffer_oop);
 819           mem->init_req(2, kit.merged_memory());
 820 
 821           // Update oop input to buffer
 822           kit.gvn().hash_delete(vt);
 823           vt->set_oop(kit.gvn(), kit.gvn().transform(oop));
 824           vt->set_is_buffered(kit.gvn());
 825           vt = kit.gvn().transform(vt)->as_InlineType();
 826 
 827           kit.set_control(kit.gvn().transform(region));
 828           kit.set_all_memory(kit.gvn().transform(mem));
 829           kit.record_for_igvn(region);
 830           kit.record_for_igvn(oop);
 831           kit.record_for_igvn(mem);
 832         }
 833         result = vt;
 834       }
 835       DEBUG_ONLY(buffer_oop = nullptr);
 836     } else {
 837       assert(result->is_top() || !call->tf()->returns_inline_type_as_fields(), "Unexpected return value");
 838     }
 839     assert(buffer_oop == nullptr, "unused buffer allocation");
 840 
 841     kit.replace_call(call, result, true, do_asserts);
 842   }
 843 }
 844 
 845 class LateInlineStringCallGenerator : public LateInlineCallGenerator {
 846 
 847  public:
 848   LateInlineStringCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
 849     LateInlineCallGenerator(method, inline_cg) {}
 850 
 851   virtual JVMState* generate(JVMState* jvms) {
 852     Compile *C = Compile::current();
 853 
 854     C->log_inline_id(this);
 855 
 856     C->add_string_late_inline(this);
 857 
 858     JVMState* new_jvms = DirectCallGenerator::generate(jvms);
 859     return new_jvms;
 860   }

1049     // Inline failed, so make a direct call.
1050     assert(_if_hit->is_inline(), "must have been a failed inline");
1051     CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method());
1052     new_jvms = cg->generate(kit.sync_jvms());
1053   }
1054   kit.add_exception_states_from(new_jvms);
1055   kit.set_jvms(new_jvms);
1056 
1057   // Need to merge slow and fast?
1058   if (slow_map == nullptr) {
1059     // The fast path is the only path remaining.
1060     return kit.transfer_exceptions_into_jvms();
1061   }
1062 
1063   if (kit.stopped()) {
1064     // Inlined method threw an exception, so it's just the slow path after all.
1065     kit.set_jvms(slow_jvms);
1066     return kit.transfer_exceptions_into_jvms();
1067   }
1068 
1069   // Allocate inline types if they are merged with objects (similar to Parse::merge_common())
1070   uint tos = kit.jvms()->stkoff() + kit.sp();
1071   uint limit = slow_map->req();
1072   for (uint i = TypeFunc::Parms; i < limit; i++) {
1073     Node* m = kit.map()->in(i);
1074     Node* n = slow_map->in(i);
1075     const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
1076     // TODO 8284443 still needed?
1077     if (m->is_InlineType() && !t->is_inlinetypeptr()) {
1078       // Allocate inline type in fast path
1079       m = m->as_InlineType()->buffer(&kit);
1080       kit.map()->set_req(i, m);
1081     }
1082     if (n->is_InlineType() && !t->is_inlinetypeptr()) {
1083       // Allocate inline type in slow path
1084       PreserveJVMState pjvms(&kit);
1085       kit.set_map(slow_map);
1086       n = n->as_InlineType()->buffer(&kit);
1087       kit.map()->set_req(i, n);
1088       slow_map = kit.stop();
1089     }
1090   }
1091 
1092   // There are 2 branches and the replaced nodes are only valid on
1093   // one: restore the replaced nodes to what they were before the
1094   // branch.
1095   kit.map()->set_replaced_nodes(replaced_nodes);
1096 
1097   // Finish the diamond.
1098   kit.C->set_has_split_ifs(true); // Has chance for split-if optimization
1099   RegionNode* region = new RegionNode(3);
1100   region->init_req(1, kit.control());
1101   region->init_req(2, slow_map->control());
1102   kit.set_control(gvn.transform(region));
1103   Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO);
1104   iophi->set_req(2, slow_map->i_o());
1105   kit.set_i_o(gvn.transform(iophi));
1106   // Merge memory
1107   kit.merge_memory(slow_map->merged_memory(), region, 2);
1108   // Transform new memory Phis.
1109   for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) {
1110     Node* phi = mms.memory();
1111     if (phi->is_Phi() && phi->in(0) == region) {
1112       mms.set_memory(gvn.transform(phi));
1113     }
1114   }


1115   for (uint i = TypeFunc::Parms; i < limit; i++) {
1116     // Skip unused stack slots; fast forward to monoff();
1117     if (i == tos) {
1118       i = kit.jvms()->monoff();
1119       if( i >= limit ) break;
1120     }
1121     Node* m = kit.map()->in(i);
1122     Node* n = slow_map->in(i);
1123     if (m != n) {
1124       const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
1125       Node* phi = PhiNode::make(region, m, t);
1126       phi->set_req(2, n);
1127       kit.map()->set_req(i, gvn.transform(phi));
1128     }
1129   }
1130   return kit.transfer_exceptions_into_jvms();
1131 }
1132 
1133 
1134 CallGenerator* CallGenerator::for_method_handle_call(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline) {
1135   assert(callee->is_method_handle_intrinsic(), "for_method_handle_call mismatch");
1136   bool input_not_const;
1137   CallGenerator* cg = CallGenerator::for_method_handle_inline(jvms, caller, callee, allow_inline, input_not_const);
1138   Compile* C = Compile::current();
1139   bool should_delay = C->should_delay_inlining();
1140   if (cg != nullptr) {
1141     if (should_delay && IncrementalInlineMH) {
1142       return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
1143     } else {
1144       return cg;
1145     }
1146   }
1147   int bci = jvms->bci();
1148   ciCallProfile profile = caller->call_profile_at_bci(bci);
1149   int call_site_count = caller->scale_count(profile.count());
1150 
1151   if (IncrementalInlineMH && (AlwaysIncrementalInline ||
1152                             (call_site_count > 0 && (should_delay || input_not_const || !C->inlining_incrementally() || C->over_inlining_cutoff())))) {
1153     return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
1154   } else {
1155     // Out-of-line call.
1156     return CallGenerator::for_direct_call(callee);
1157   }
1158 }
1159 
1160 
1161 CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline, bool& input_not_const) {
1162   GraphKit kit(jvms);
1163   PhaseGVN& gvn = kit.gvn();
1164   Compile* C = kit.C;
1165   vmIntrinsics::ID iid = callee->intrinsic_id();
1166   input_not_const = true;
1167   if (StressMethodHandleLinkerInlining) {
1168     allow_inline = false;
1169   }
1170   switch (iid) {
1171   case vmIntrinsics::_invokeBasic:
1172     {
1173       // Get MethodHandle receiver:
1174       Node* receiver = kit.argument(0);
1175       if (receiver->Opcode() == Op_ConP) {
1176         input_not_const = false;
1177         const TypeOopPtr* recv_toop = receiver->bottom_type()->isa_oopptr();
1178         if (recv_toop != nullptr) {
1179           ciMethod* target = recv_toop->const_oop()->as_method_handle()->get_vmtarget();
1180           const int vtable_index = Method::invalid_vtable_index;

1188                                                 false /* call_does_dispatch */,
1189                                                 jvms,
1190                                                 allow_inline,
1191                                                 PROB_ALWAYS);
1192           return cg;
1193         } else {
1194           assert(receiver->bottom_type() == TypePtr::NULL_PTR, "not a null: %s",
1195                  Type::str(receiver->bottom_type()));
1196           print_inlining_failure(C, callee, jvms, "receiver is always null");
1197         }
1198       } else {
1199         print_inlining_failure(C, callee, jvms, "receiver not constant");
1200       }
1201   } break;
1202 
1203   case vmIntrinsics::_linkToVirtual:
1204   case vmIntrinsics::_linkToStatic:
1205   case vmIntrinsics::_linkToSpecial:
1206   case vmIntrinsics::_linkToInterface:
1207     {
1208       int nargs = callee->arg_size();
1209       // Get MemberName argument:
1210       Node* member_name = kit.argument(nargs - 1);
1211       if (member_name->Opcode() == Op_ConP) {
1212         input_not_const = false;
1213         const TypeOopPtr* oop_ptr = member_name->bottom_type()->is_oopptr();
1214         ciMethod* target = oop_ptr->const_oop()->as_member_name()->get_vmtarget();
1215 
1216         if (!ciMethod::is_consistent_info(callee, target)) {
1217           print_inlining_failure(C, callee, jvms, "signatures mismatch");
1218           return nullptr;
1219         }
1220 
1221         // In lambda forms we erase signature types to avoid resolving issues
1222         // involving class loaders.  When we optimize a method handle invoke
1223         // to a direct call we must cast the receiver and arguments to its
1224         // actual types.
1225         ciSignature* signature = target->signature();
1226         const int receiver_skip = target->is_static() ? 0 : 1;
1227         // Cast receiver to its type.
1228         if (!target->is_static()) {
1229           Node* recv = kit.argument(0);
1230           Node* casted_recv = kit.maybe_narrow_object_type(recv, signature->accessing_klass());

1260         ciKlass* speculative_receiver_type = nullptr;
1261         if (is_virtual_or_interface) {
1262           ciInstanceKlass* klass = target->holder();
1263           Node*             receiver_node = kit.argument(0);
1264           const TypeOopPtr* receiver_type = gvn.type(receiver_node)->isa_oopptr();
1265           // call_does_dispatch and vtable_index are out-parameters.  They might be changed.
1266           // optimize_virtual_call() takes 2 different holder
1267           // arguments for a corner case that doesn't apply here (see
1268           // Parse::do_call())
1269           target = C->optimize_virtual_call(caller, klass, klass,
1270                                             target, receiver_type, is_virtual,
1271                                             call_does_dispatch, vtable_index, // out-parameters
1272                                             false /* check_access */);
1273           // We lack profiling at this call but type speculation may
1274           // provide us with a type
1275           speculative_receiver_type = (receiver_type != nullptr) ? receiver_type->speculative_type() : nullptr;
1276         }
1277         CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms,
1278                                               allow_inline,
1279                                               PROB_ALWAYS,
1280                                               speculative_receiver_type,
1281                                               true);
1282         return cg;
1283       } else {
1284         print_inlining_failure(C, callee, jvms, "member_name not constant");
1285       }
1286   } break;
1287 
1288   case vmIntrinsics::_linkToNative:
1289     print_inlining_failure(C, callee, jvms, "native call");
1290     break;
1291 
1292   default:
1293     fatal("unexpected intrinsic %d: %s", vmIntrinsics::as_int(iid), vmIntrinsics::name_at(iid));
1294     break;
1295   }
1296   return nullptr;
1297 }
1298 
1299 //------------------------PredicatedIntrinsicGenerator------------------------------
1300 // Internal class which handles all predicated Intrinsic calls.
1301 class PredicatedIntrinsicGenerator : public CallGenerator {

1333   //        do_intrinsic(0)
1334   //    else
1335   //    if (predicate(1))
1336   //        do_intrinsic(1)
1337   //    ...
1338   //    else
1339   //        do_java_comp
1340 
1341   GraphKit kit(jvms);
1342   PhaseGVN& gvn = kit.gvn();
1343 
1344   CompileLog* log = kit.C->log();
1345   if (log != nullptr) {
1346     log->elem("predicated_intrinsic bci='%d' method='%d'",
1347               jvms->bci(), log->identify(method()));
1348   }
1349 
1350   if (!method()->is_static()) {
1351     // We need an explicit receiver null_check before checking its type in predicate.
1352     // We share a map with the caller, so his JVMS gets adjusted.
1353     kit.null_check_receiver_before_call(method());
1354     if (kit.stopped()) {
1355       return kit.transfer_exceptions_into_jvms();
1356     }
1357   }
1358 
1359   int n_predicates = _intrinsic->predicates_count();
1360   assert(n_predicates > 0, "sanity");
1361 
1362   JVMState** result_jvms = NEW_RESOURCE_ARRAY(JVMState*, (n_predicates+1));
1363 
1364   // Region for normal compilation code if intrinsic failed.
1365   Node* slow_region = new RegionNode(1);
1366 
1367   int results = 0;
1368   for (int predicate = 0; (predicate < n_predicates) && !kit.stopped(); predicate++) {
1369 #ifdef ASSERT
1370     JVMState* old_jvms = kit.jvms();
1371     SafePointNode* old_map = kit.map();
1372     Node* old_io  = old_map->i_o();
1373     Node* old_mem = old_map->memory();
< prev index next >