1 /*
   2  * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "ci/bcEscapeAnalyzer.hpp"
  27 #include "ci/ciCallSite.hpp"
  28 #include "ci/ciObjArray.hpp"
  29 #include "ci/ciMemberName.hpp"
  30 #include "ci/ciMethodHandle.hpp"
  31 #include "classfile/javaClasses.hpp"
  32 #include "compiler/compileLog.hpp"
  33 #include "opto/addnode.hpp"
  34 #include "opto/callGenerator.hpp"
  35 #include "opto/callnode.hpp"
  36 #include "opto/castnode.hpp"
  37 #include "opto/cfgnode.hpp"
  38 #include "opto/parse.hpp"
  39 #include "opto/rootnode.hpp"
  40 #include "opto/runtime.hpp"
  41 #include "opto/subnode.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "ci/ciNativeEntryPoint.hpp"
  44 #include "utilities/debug.hpp"
  45 
  46 // Utility function.
  47 const TypeFunc* CallGenerator::tf() const {
  48   return TypeFunc::make(method());
  49 }
  50 
  51 bool CallGenerator::is_inlined_method_handle_intrinsic(JVMState* jvms, ciMethod* m) {
  52   return is_inlined_method_handle_intrinsic(jvms->method(), jvms->bci(), m);
  53 }
  54 
  55 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* caller, int bci, ciMethod* m) {
  56   ciMethod* symbolic_info = caller->get_method_at_bci(bci);
  57   return is_inlined_method_handle_intrinsic(symbolic_info, m);
  58 }
  59 
  60 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* symbolic_info, ciMethod* m) {
  61   return symbolic_info->is_method_handle_intrinsic() && !m->is_method_handle_intrinsic();
  62 }
  63 
  64 //-----------------------------ParseGenerator---------------------------------
  65 // Internal class which handles all direct bytecode traversal.
  66 class ParseGenerator : public InlineCallGenerator {
  67 private:
  68   bool  _is_osr;
  69   float _expected_uses;
  70 
  71 public:
  72   ParseGenerator(ciMethod* method, float expected_uses, bool is_osr = false)
  73     : InlineCallGenerator(method)
  74   {
  75     _is_osr        = is_osr;
  76     _expected_uses = expected_uses;
  77     assert(InlineTree::check_can_parse(method) == NULL, "parse must be possible");
  78   }
  79 
  80   virtual bool      is_parse() const           { return true; }
  81   virtual JVMState* generate(JVMState* jvms);
  82   int is_osr() { return _is_osr; }
  83 
  84 };
  85 
  86 JVMState* ParseGenerator::generate(JVMState* jvms) {
  87   Compile* C = Compile::current();
  88   C->print_inlining_update(this);
  89 
  90   if (is_osr()) {
  91     // The JVMS for a OSR has a single argument (see its TypeFunc).
  92     assert(jvms->depth() == 1, "no inline OSR");
  93   }
  94 
  95   if (C->failing()) {
  96     return NULL;  // bailing out of the compile; do not try to parse
  97   }
  98 
  99   Parse parser(jvms, method(), _expected_uses);
 100   // Grab signature for matching/allocation
 101   GraphKit& exits = parser.exits();
 102 
 103   if (C->failing()) {
 104     while (exits.pop_exception_state() != NULL) ;
 105     return NULL;
 106   }
 107 
 108   assert(exits.jvms()->same_calls_as(jvms), "sanity");
 109 
 110   // Simply return the exit state of the parser,
 111   // augmented by any exceptional states.
 112   return exits.transfer_exceptions_into_jvms();
 113 }
 114 
 115 //---------------------------DirectCallGenerator------------------------------
 116 // Internal class which handles all out-of-line calls w/o receiver type checks.
 117 class DirectCallGenerator : public CallGenerator {
 118  private:
 119   CallStaticJavaNode* _call_node;
 120   // Force separate memory and I/O projections for the exceptional
 121   // paths to facilitate late inlinig.
 122   bool                _separate_io_proj;
 123 
 124 protected:
 125   void set_call_node(CallStaticJavaNode* call) { _call_node = call; }
 126 
 127  public:
 128   DirectCallGenerator(ciMethod* method, bool separate_io_proj)
 129     : CallGenerator(method),
 130       _separate_io_proj(separate_io_proj)
 131   {
 132   }
 133   virtual JVMState* generate(JVMState* jvms);
 134 
 135   virtual CallNode* call_node() const { return _call_node; }
 136   virtual CallGenerator* with_call_node(CallNode* call) {
 137     DirectCallGenerator* dcg = new DirectCallGenerator(method(), _separate_io_proj);
 138     dcg->set_call_node(call->as_CallStaticJava());
 139     return dcg;
 140   }
 141 };
 142 
 143 JVMState* DirectCallGenerator::generate(JVMState* jvms) {
 144   GraphKit kit(jvms);
 145   kit.C->print_inlining_update(this);
 146   bool is_static = method()->is_static();
 147   address target = is_static ? SharedRuntime::get_resolve_static_call_stub()
 148                              : SharedRuntime::get_resolve_opt_virtual_call_stub();
 149 
 150   if (kit.C->log() != NULL) {
 151     kit.C->log()->elem("direct_call bci='%d'", jvms->bci());
 152   }
 153 
 154   CallStaticJavaNode* call = new CallStaticJavaNode(kit.C, tf(), target, method());
 155   if (is_inlined_method_handle_intrinsic(jvms, method())) {
 156     // To be able to issue a direct call and skip a call to MH.linkTo*/invokeBasic adapter,
 157     // additional information about the method being invoked should be attached
 158     // to the call site to make resolution logic work
 159     // (see SharedRuntime::resolve_static_call_C).
 160     call->set_override_symbolic_info(true);
 161   }
 162   _call_node = call;  // Save the call node in case we need it later
 163   if (!is_static) {
 164     // Make an explicit receiver null_check as part of this call.
 165     // Since we share a map with the caller, his JVMS gets adjusted.
 166     kit.null_check_receiver_before_call(method());
 167     if (kit.stopped()) {
 168       // And dump it back to the caller, decorated with any exceptions:
 169       return kit.transfer_exceptions_into_jvms();
 170     }
 171     // Mark the call node as virtual, sort of:
 172     call->set_optimized_virtual(true);
 173     if (method()->is_method_handle_intrinsic() ||
 174         method()->is_compiled_lambda_form()) {
 175       call->set_method_handle_invoke(true);
 176     }
 177   }
 178   kit.set_arguments_for_java_call(call);
 179   kit.set_edges_for_java_call(call, false, _separate_io_proj);
 180   Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
 181   kit.push_node(method()->return_type()->basic_type(), ret);
 182   return kit.transfer_exceptions_into_jvms();
 183 }
 184 
 185 //--------------------------VirtualCallGenerator------------------------------
 186 // Internal class which handles all out-of-line calls checking receiver type.
 187 class VirtualCallGenerator : public CallGenerator {
 188 private:
 189   int _vtable_index;
 190   bool _separate_io_proj;
 191   CallDynamicJavaNode* _call_node;
 192 
 193 protected:
 194   void set_call_node(CallDynamicJavaNode* call) { _call_node = call; }
 195 
 196 public:
 197   VirtualCallGenerator(ciMethod* method, int vtable_index, bool separate_io_proj)
 198     : CallGenerator(method), _vtable_index(vtable_index), _separate_io_proj(separate_io_proj), _call_node(NULL)
 199   {
 200     assert(vtable_index == Method::invalid_vtable_index ||
 201            vtable_index >= 0, "either invalid or usable");
 202   }
 203   virtual bool      is_virtual() const          { return true; }
 204   virtual JVMState* generate(JVMState* jvms);
 205 
 206   virtual CallNode* call_node() const { return _call_node; }
 207   int vtable_index() const { return _vtable_index; }
 208 
 209   virtual CallGenerator* with_call_node(CallNode* call) {
 210     VirtualCallGenerator* cg = new VirtualCallGenerator(method(), _vtable_index, _separate_io_proj);
 211     cg->set_call_node(call->as_CallDynamicJava());
 212     return cg;
 213   }
 214 };
 215 
 216 JVMState* VirtualCallGenerator::generate(JVMState* jvms) {
 217   GraphKit kit(jvms);
 218   Node* receiver = kit.argument(0);
 219 
 220   kit.C->print_inlining_update(this);
 221 
 222   if (kit.C->log() != NULL) {
 223     kit.C->log()->elem("virtual_call bci='%d'", jvms->bci());
 224   }
 225 
 226   // If the receiver is a constant null, do not torture the system
 227   // by attempting to call through it.  The compile will proceed
 228   // correctly, but may bail out in final_graph_reshaping, because
 229   // the call instruction will have a seemingly deficient out-count.
 230   // (The bailout says something misleading about an "infinite loop".)
 231   if (kit.gvn().type(receiver)->higher_equal(TypePtr::NULL_PTR)) {
 232     assert(Bytecodes::is_invoke(kit.java_bc()), "%d: %s", kit.java_bc(), Bytecodes::name(kit.java_bc()));
 233     ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci());
 234     int arg_size = declared_method->signature()->arg_size_for_bc(kit.java_bc());
 235     kit.inc_sp(arg_size);  // restore arguments
 236     kit.uncommon_trap(Deoptimization::Reason_null_check,
 237                       Deoptimization::Action_none,
 238                       NULL, "null receiver");
 239     return kit.transfer_exceptions_into_jvms();
 240   }
 241 
 242   // Ideally we would unconditionally do a null check here and let it
 243   // be converted to an implicit check based on profile information.
 244   // However currently the conversion to implicit null checks in
 245   // Block::implicit_null_check() only looks for loads and stores, not calls.
 246   ciMethod *caller = kit.method();
 247   ciMethodData *caller_md = (caller == NULL) ? NULL : caller->method_data();
 248   if (!UseInlineCaches || !ImplicitNullChecks || !os::zero_page_read_protected() ||
 249        ((ImplicitNullCheckThreshold > 0) && caller_md &&
 250        (caller_md->trap_count(Deoptimization::Reason_null_check)
 251        >= (uint)ImplicitNullCheckThreshold))) {
 252     // Make an explicit receiver null_check as part of this call.
 253     // Since we share a map with the caller, his JVMS gets adjusted.
 254     receiver = kit.null_check_receiver_before_call(method());
 255     if (kit.stopped()) {
 256       // And dump it back to the caller, decorated with any exceptions:
 257       return kit.transfer_exceptions_into_jvms();
 258     }
 259   }
 260 
 261   assert(!method()->is_static(), "virtual call must not be to static");
 262   assert(!method()->is_final(), "virtual call should not be to final");
 263   assert(!method()->is_private(), "virtual call should not be to private");
 264   assert(_vtable_index == Method::invalid_vtable_index || !UseInlineCaches,
 265          "no vtable calls if +UseInlineCaches ");
 266   address target = SharedRuntime::get_resolve_virtual_call_stub();
 267   // Normal inline cache used for call
 268   CallDynamicJavaNode* call = new CallDynamicJavaNode(tf(), target, method(), _vtable_index);
 269   if (is_inlined_method_handle_intrinsic(jvms, method())) {
 270     // To be able to issue a direct call (optimized virtual or virtual)
 271     // and skip a call to MH.linkTo*/invokeBasic adapter, additional information
 272     // about the method being invoked should be attached to the call site to
 273     // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C).
 274     call->set_override_symbolic_info(true);
 275   }
 276   _call_node = call;  // Save the call node in case we need it later
 277 
 278   kit.set_arguments_for_java_call(call);
 279   kit.set_edges_for_java_call(call, false /*must_throw*/, _separate_io_proj);
 280   Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
 281   kit.push_node(method()->return_type()->basic_type(), ret);
 282 
 283   // Represent the effect of an implicit receiver null_check
 284   // as part of this call.  Since we share a map with the caller,
 285   // his JVMS gets adjusted.
 286   kit.cast_not_null(receiver);
 287   return kit.transfer_exceptions_into_jvms();
 288 }
 289 
 290 CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) {
 291   if (InlineTree::check_can_parse(m) != NULL)  return NULL;
 292   return new ParseGenerator(m, expected_uses);
 293 }
 294 
 295 // As a special case, the JVMS passed to this CallGenerator is
 296 // for the method execution already in progress, not just the JVMS
 297 // of the caller.  Thus, this CallGenerator cannot be mixed with others!
 298 CallGenerator* CallGenerator::for_osr(ciMethod* m, int osr_bci) {
 299   if (InlineTree::check_can_parse(m) != NULL)  return NULL;
 300   float past_uses = m->interpreter_invocation_count();
 301   float expected_uses = past_uses;
 302   return new ParseGenerator(m, expected_uses, true);
 303 }
 304 
 305 CallGenerator* CallGenerator::for_direct_call(ciMethod* m, bool separate_io_proj) {
 306   assert(!m->is_abstract(), "for_direct_call mismatch");
 307   return new DirectCallGenerator(m, separate_io_proj);
 308 }
 309 
 310 CallGenerator* CallGenerator::for_virtual_call(ciMethod* m, int vtable_index) {
 311   assert(!m->is_static(), "for_virtual_call mismatch");
 312   assert(!m->is_method_handle_intrinsic(), "should be a direct call");
 313   return new VirtualCallGenerator(m, vtable_index, false /*separate_io_projs*/);
 314 }
 315 
 316 // Allow inlining decisions to be delayed
 317 class LateInlineCallGenerator : public DirectCallGenerator {
 318  private:
 319   jlong _unique_id;   // unique id for log compilation
 320   bool _is_pure_call; // a hint that the call doesn't have important side effects to care about
 321 
 322  protected:
 323   CallGenerator* _inline_cg;
 324   virtual bool do_late_inline_check(Compile* C, JVMState* jvms) { return true; }
 325   virtual CallGenerator* inline_cg() const { return _inline_cg; }
 326   virtual bool is_pure_call() const { return _is_pure_call; }
 327 
 328  public:
 329   LateInlineCallGenerator(ciMethod* method, CallGenerator* inline_cg, bool is_pure_call = false) :
 330     DirectCallGenerator(method, true), _unique_id(0), _is_pure_call(is_pure_call), _inline_cg(inline_cg) {}
 331 
 332   virtual bool is_late_inline() const { return true; }
 333 
 334   // Convert the CallStaticJava into an inline
 335   virtual void do_late_inline();
 336 
 337   virtual JVMState* generate(JVMState* jvms) {
 338     Compile *C = Compile::current();
 339 
 340     C->log_inline_id(this);
 341 
 342     // Record that this call site should be revisited once the main
 343     // parse is finished.
 344     if (!is_mh_late_inline()) {
 345       C->add_late_inline(this);
 346     }
 347 
 348     // Emit the CallStaticJava and request separate projections so
 349     // that the late inlining logic can distinguish between fall
 350     // through and exceptional uses of the memory and io projections
 351     // as is done for allocations and macro expansion.
 352     return DirectCallGenerator::generate(jvms);
 353   }
 354 
 355   virtual void print_inlining_late(const char* msg) {
 356     CallNode* call = call_node();
 357     Compile* C = Compile::current();
 358     C->print_inlining_assert_ready();
 359     C->print_inlining(method(), call->jvms()->depth()-1, call->jvms()->bci(), msg);
 360     C->print_inlining_move_to(this);
 361     C->print_inlining_update_delayed(this);
 362   }
 363 
 364   virtual void set_unique_id(jlong id) {
 365     _unique_id = id;
 366   }
 367 
 368   virtual jlong unique_id() const {
 369     return _unique_id;
 370   }
 371 
 372   virtual CallGenerator* with_call_node(CallNode* call) {
 373     LateInlineCallGenerator* cg = new LateInlineCallGenerator(method(), _inline_cg, _is_pure_call);
 374     cg->set_call_node(call->as_CallStaticJava());
 375     return cg;
 376   }
 377 };
 378 
 379 CallGenerator* CallGenerator::for_late_inline(ciMethod* method, CallGenerator* inline_cg) {
 380   return new LateInlineCallGenerator(method, inline_cg);
 381 }
 382 
 383 class LateInlineMHCallGenerator : public LateInlineCallGenerator {
 384   ciMethod* _caller;
 385   bool _input_not_const;
 386 
 387   virtual bool do_late_inline_check(Compile* C, JVMState* jvms);
 388 
 389  public:
 390   LateInlineMHCallGenerator(ciMethod* caller, ciMethod* callee, bool input_not_const) :
 391     LateInlineCallGenerator(callee, NULL), _caller(caller), _input_not_const(input_not_const) {}
 392 
 393   virtual bool is_mh_late_inline() const { return true; }
 394 
 395   // Convert the CallStaticJava into an inline
 396   virtual void do_late_inline();
 397 
 398   virtual JVMState* generate(JVMState* jvms) {
 399     JVMState* new_jvms = LateInlineCallGenerator::generate(jvms);
 400 
 401     Compile* C = Compile::current();
 402     if (_input_not_const) {
 403       // inlining won't be possible so no need to enqueue right now.
 404       call_node()->set_generator(this);
 405     } else {
 406       C->add_late_inline(this);
 407     }
 408     return new_jvms;
 409   }
 410 
 411   virtual CallGenerator* with_call_node(CallNode* call) {
 412     LateInlineMHCallGenerator* cg = new LateInlineMHCallGenerator(_caller, method(), _input_not_const);
 413     cg->set_call_node(call->as_CallStaticJava());
 414     return cg;
 415   }
 416 };
 417 
 418 bool LateInlineMHCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) {
 419   // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call.
 420   bool allow_inline = C->inlining_incrementally();
 421   bool input_not_const = true;
 422   CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), allow_inline, input_not_const);
 423   assert(!input_not_const, "sanity"); // shouldn't have been scheduled for inlining in the first place
 424 
 425   if (cg != NULL) {
 426     assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline, "we're doing late inlining");
 427     _inline_cg = cg;
 428     C->dec_number_of_mh_late_inlines();
 429     return true;
 430   } else {
 431     // Method handle call which has a constant appendix argument should be either inlined or replaced with a direct call
 432     // unless there's a signature mismatch between caller and callee. If the failure occurs, there's not much to be improved later,
 433     // so don't reinstall the generator to avoid pushing the generator between IGVN and incremental inlining indefinitely.
 434     return false;
 435   }
 436 }
 437 
 438 CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const) {
 439   assert(IncrementalInlineMH, "required");
 440   Compile::current()->inc_number_of_mh_late_inlines();
 441   CallGenerator* cg = new LateInlineMHCallGenerator(caller, callee, input_not_const);
 442   return cg;
 443 }
 444 
 445 // Allow inlining decisions to be delayed
 446 class LateInlineVirtualCallGenerator : public VirtualCallGenerator {
 447  private:
 448   jlong          _unique_id;   // unique id for log compilation
 449   CallGenerator* _inline_cg;
 450   ciMethod*      _callee;
 451   bool           _is_pure_call;
 452   float          _prof_factor;
 453 
 454  protected:
 455   virtual bool do_late_inline_check(Compile* C, JVMState* jvms);
 456   virtual CallGenerator* inline_cg() const { return _inline_cg; }
 457   virtual bool is_pure_call() const { return _is_pure_call; }
 458 
 459  public:
 460   LateInlineVirtualCallGenerator(ciMethod* method, int vtable_index, float prof_factor)
 461   : VirtualCallGenerator(method, vtable_index, true /*separate_io_projs*/),
 462     _unique_id(0), _inline_cg(NULL), _callee(NULL), _is_pure_call(false), _prof_factor(prof_factor) {
 463     assert(IncrementalInlineVirtual, "required");
 464   }
 465 
 466   virtual bool is_late_inline() const { return true; }
 467 
 468   virtual bool is_virtual_late_inline() const { return true; }
 469 
 470   // Convert the CallDynamicJava into an inline
 471   virtual void do_late_inline();
 472 
 473   virtual void set_callee_method(ciMethod* m) {
 474     assert(_callee == NULL, "repeated inlining attempt");
 475     _callee = m;
 476   }
 477 
 478   virtual JVMState* generate(JVMState* jvms) {
 479     // Emit the CallDynamicJava and request separate projections so
 480     // that the late inlining logic can distinguish between fall
 481     // through and exceptional uses of the memory and io projections
 482     // as is done for allocations and macro expansion.
 483     JVMState* new_jvms = VirtualCallGenerator::generate(jvms);
 484     if (call_node() != NULL) {
 485       call_node()->set_generator(this);
 486     }
 487     return new_jvms;
 488   }
 489 
 490   virtual void print_inlining_late(const char* msg) {
 491     CallNode* call = call_node();
 492     Compile* C = Compile::current();
 493     C->print_inlining_assert_ready();
 494     C->print_inlining(method(), call->jvms()->depth()-1, call->jvms()->bci(), msg);
 495     C->print_inlining_move_to(this);
 496     C->print_inlining_update_delayed(this);
 497   }
 498 
 499   virtual void set_unique_id(jlong id) {
 500     _unique_id = id;
 501   }
 502 
 503   virtual jlong unique_id() const {
 504     return _unique_id;
 505   }
 506 
 507   virtual CallGenerator* with_call_node(CallNode* call) {
 508     LateInlineVirtualCallGenerator* cg = new LateInlineVirtualCallGenerator(method(), vtable_index(), _prof_factor);
 509     cg->set_call_node(call->as_CallDynamicJava());
 510     return cg;
 511   }
 512 };
 513 
 514 bool LateInlineVirtualCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) {
 515   // Method handle linker case is handled in CallDynamicJavaNode::Ideal().
 516   // Unless inlining is performed, _override_symbolic_info bit will be set in DirectCallGenerator::generate().
 517 
 518   // Implicit receiver null checks introduce problems when exception states are combined.
 519   Node* receiver = jvms->map()->argument(jvms, 0);
 520   const Type* recv_type = C->initial_gvn()->type(receiver);
 521   if (recv_type->maybe_null()) {
 522     return false;
 523   }
 524   // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call.
 525   bool allow_inline = C->inlining_incrementally();
 526   if (!allow_inline && _callee->holder()->is_interface()) {
 527     // Don't convert the interface call to a direct call guarded by an interface subtype check.
 528     return false;
 529   }
 530   CallGenerator* cg = C->call_generator(_callee,
 531                                         vtable_index(),
 532                                         false /*call_does_dispatch*/,
 533                                         jvms,
 534                                         allow_inline,
 535                                         _prof_factor,
 536                                         NULL /*speculative_receiver_type*/,
 537                                         true /*allow_intrinsics*/);
 538 
 539   if (cg != NULL) {
 540     assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline, "we're doing late inlining");
 541     _inline_cg = cg;
 542     return true;
 543   } else {
 544     // Virtual call which provably doesn't dispatch should be either inlined or replaced with a direct call.
 545     assert(false, "no progress");
 546     return false;
 547   }
 548 }
 549 
 550 CallGenerator* CallGenerator::for_late_inline_virtual(ciMethod* m, int vtable_index, float prof_factor) {
 551   assert(IncrementalInlineVirtual, "required");
 552   assert(!m->is_static(), "for_virtual_call mismatch");
 553   assert(!m->is_method_handle_intrinsic(), "should be a direct call");
 554   return new LateInlineVirtualCallGenerator(m, vtable_index, prof_factor);
 555 }
 556 
 557 void LateInlineCallGenerator::do_late_inline() {
 558   CallGenerator::do_late_inline_helper();
 559 }
 560 
 561 void LateInlineMHCallGenerator::do_late_inline() {
 562   CallGenerator::do_late_inline_helper();
 563 }
 564 
 565 void LateInlineVirtualCallGenerator::do_late_inline() {
 566   assert(_callee != NULL, "required"); // set up in CallDynamicJavaNode::Ideal
 567   CallGenerator::do_late_inline_helper();
 568 }
 569 
 570 static bool has_non_debug_usages(Node* n) {
 571   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
 572     Node* m = n->fast_out(i);
 573     if (!m->is_SafePoint()
 574         || (m->is_Call() && m->as_Call()->has_non_debug_use(n))) {
 575       return true;
 576     }
 577   }
 578   return false;
 579 }
 580 
 581 static bool is_box_cache_valid(CallNode* call) {
 582   ciInstanceKlass* klass = call->as_CallStaticJava()->method()->holder();
 583   return klass->is_box_cache_valid();
 584 }
 585 
 586 // delay box in runtime, treat box as a scalarized object
 587 static void scalarize_debug_usages(CallNode* call, Node* resproj) {
 588   GraphKit kit(call->jvms());
 589   PhaseGVN& gvn = kit.gvn();
 590 
 591   ProjNode* res = resproj->as_Proj();
 592   ciInstanceKlass* klass = call->as_CallStaticJava()->method()->holder();
 593   int n_fields = klass->nof_nonstatic_fields();
 594   assert(n_fields == 1, "the klass must be an auto-boxing klass");
 595 
 596   for (DUIterator_Last imin, i = res->last_outs(imin); i >= imin;) {
 597     SafePointNode* sfpt = res->last_out(i)->as_SafePoint();
 598     uint first_ind = sfpt->req() - sfpt->jvms()->scloff();
 599     Node* sobj = new SafePointScalarObjectNode(gvn.type(res)->isa_oopptr(),
 600 #ifdef ASSERT
 601                                                 call,
 602 #endif // ASSERT
 603                                                 first_ind, n_fields, true);
 604     sobj->init_req(0, kit.root());
 605     sfpt->add_req(call->in(TypeFunc::Parms));
 606     sobj = gvn.transform(sobj);
 607     JVMState* jvms = sfpt->jvms();
 608     jvms->set_endoff(sfpt->req());
 609     int start = jvms->debug_start();
 610     int end   = jvms->debug_end();
 611     int num_edges = sfpt->replace_edges_in_range(res, sobj, start, end, &gvn);
 612     i -= num_edges;
 613   }
 614 
 615   assert(res->outcnt() == 0, "the box must have no use after replace");
 616 
 617 #ifndef PRODUCT
 618   if (PrintEliminateAllocations) {
 619     tty->print("++++ Eliminated: %d ", call->_idx);
 620     call->as_CallStaticJava()->method()->print_short_name(tty);
 621     tty->cr();
 622   }
 623 #endif
 624 }
 625 
 626 void CallGenerator::do_late_inline_helper() {
 627   assert(is_late_inline(), "only late inline allowed");
 628 
 629   // Can't inline it
 630   CallNode* call = call_node();
 631   if (call == NULL || call->outcnt() == 0 ||
 632       call->in(0) == NULL || call->in(0)->is_top()) {
 633     return;
 634   }
 635 
 636   const TypeTuple *r = call->tf()->domain();
 637   for (int i1 = 0; i1 < method()->arg_size(); i1++) {
 638     if (call->in(TypeFunc::Parms + i1)->is_top() && r->field_at(TypeFunc::Parms + i1) != Type::HALF) {
 639       assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
 640       return;
 641     }
 642   }
 643 
 644   if (call->in(TypeFunc::Memory)->is_top()) {
 645     assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
 646     return;
 647   }
 648   if (call->in(TypeFunc::Memory)->is_MergeMem()) {
 649     MergeMemNode* merge_mem = call->in(TypeFunc::Memory)->as_MergeMem();
 650     if (merge_mem->base_memory() == merge_mem->empty_memory()) {
 651       return; // dead path
 652     }
 653   }
 654 
 655   // check for unreachable loop
 656   CallProjections callprojs;
 657   call->extract_projections(&callprojs, true);
 658   if ((callprojs.fallthrough_catchproj == call->in(0)) ||
 659       (callprojs.catchall_catchproj    == call->in(0)) ||
 660       (callprojs.fallthrough_memproj   == call->in(TypeFunc::Memory)) ||
 661       (callprojs.catchall_memproj      == call->in(TypeFunc::Memory)) ||
 662       (callprojs.fallthrough_ioproj    == call->in(TypeFunc::I_O)) ||
 663       (callprojs.catchall_ioproj       == call->in(TypeFunc::I_O)) ||
 664       (callprojs.resproj != NULL && call->find_edge(callprojs.resproj) != -1) ||
 665       (callprojs.exobj   != NULL && call->find_edge(callprojs.exobj) != -1)) {
 666     return;
 667   }
 668 
 669   Compile* C = Compile::current();
 670   // Remove inlined methods from Compiler's lists.
 671   if (call->is_macro()) {
 672     C->remove_macro_node(call);
 673   }
 674 
 675   bool result_not_used = false;
 676 
 677   if (is_pure_call()) {
 678     // Disabled due to JDK-8276112
 679     if (false && is_boxing_late_inline() && callprojs.resproj != nullptr) {
 680       // replace box node to scalar node only in case it is directly referenced by debug info
 681       assert(call->as_CallStaticJava()->is_boxing_method(), "sanity");
 682       if (!has_non_debug_usages(callprojs.resproj) && is_box_cache_valid(call)) {
 683         scalarize_debug_usages(call, callprojs.resproj);
 684       }
 685     }
 686 
 687     // The call is marked as pure (no important side effects), but result isn't used.
 688     // It's safe to remove the call.
 689     result_not_used = (callprojs.resproj == NULL || callprojs.resproj->outcnt() == 0);
 690   }
 691 
 692   if (result_not_used) {
 693     GraphKit kit(call->jvms());
 694     kit.replace_call(call, C->top(), true);
 695   } else {
 696     // Make a clone of the JVMState that appropriate to use for driving a parse
 697     JVMState* old_jvms = call->jvms();
 698     JVMState* jvms = old_jvms->clone_shallow(C);
 699     uint size = call->req();
 700     SafePointNode* map = new SafePointNode(size, jvms);
 701     for (uint i1 = 0; i1 < size; i1++) {
 702       map->init_req(i1, call->in(i1));
 703     }
 704 
 705     // Make sure the state is a MergeMem for parsing.
 706     if (!map->in(TypeFunc::Memory)->is_MergeMem()) {
 707       Node* mem = MergeMemNode::make(map->in(TypeFunc::Memory));
 708       C->initial_gvn()->set_type_bottom(mem);
 709       map->set_req(TypeFunc::Memory, mem);
 710     }
 711 
 712     uint nargs = method()->arg_size();
 713     // blow away old call arguments
 714     Node* top = C->top();
 715     for (uint i1 = 0; i1 < nargs; i1++) {
 716       map->set_req(TypeFunc::Parms + i1, top);
 717     }
 718     jvms->set_map(map);
 719 
 720     // Make enough space in the expression stack to transfer
 721     // the incoming arguments and return value.
 722     map->ensure_stack(jvms, jvms->method()->max_stack());
 723     for (uint i1 = 0; i1 < nargs; i1++) {
 724       map->set_argument(jvms, i1, call->in(TypeFunc::Parms + i1));
 725     }
 726 
 727     C->print_inlining_assert_ready();
 728 
 729     C->print_inlining_move_to(this);
 730 
 731     C->log_late_inline(this);
 732 
 733     // JVMState is ready, so time to perform some checks and prepare for inlining attempt.
 734     if (!do_late_inline_check(C, jvms)) {
 735       map->disconnect_inputs(C);
 736       C->print_inlining_update_delayed(this);
 737       return;
 738     }
 739 
 740     // Setup default node notes to be picked up by the inlining
 741     Node_Notes* old_nn = C->node_notes_at(call->_idx);
 742     if (old_nn != NULL) {
 743       Node_Notes* entry_nn = old_nn->clone(C);
 744       entry_nn->set_jvms(jvms);
 745       C->set_default_node_notes(entry_nn);
 746     }
 747 
 748     // Now perform the inlining using the synthesized JVMState
 749     JVMState* new_jvms = inline_cg()->generate(jvms);
 750     if (new_jvms == NULL)  return;  // no change
 751     if (C->failing())      return;
 752 
 753     // Capture any exceptional control flow
 754     GraphKit kit(new_jvms);
 755 
 756     // Find the result object
 757     Node* result = C->top();
 758     int   result_size = method()->return_type()->size();
 759     if (result_size != 0 && !kit.stopped()) {
 760       result = (result_size == 1) ? kit.pop() : kit.pop_pair();
 761     }
 762 
 763     if (inline_cg()->is_inline()) {
 764       C->set_has_loops(C->has_loops() || inline_cg()->method()->has_loops());
 765       C->env()->notice_inlined_method(inline_cg()->method());
 766     }
 767     C->set_inlining_progress(true);
 768     C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup
 769     kit.replace_call(call, result, true);
 770   }
 771 }
 772 
 773 class LateInlineStringCallGenerator : public LateInlineCallGenerator {
 774 
 775  public:
 776   LateInlineStringCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
 777     LateInlineCallGenerator(method, inline_cg) {}
 778 
 779   virtual JVMState* generate(JVMState* jvms) {
 780     Compile *C = Compile::current();
 781 
 782     C->log_inline_id(this);
 783 
 784     C->add_string_late_inline(this);
 785 
 786     JVMState* new_jvms = DirectCallGenerator::generate(jvms);
 787     return new_jvms;
 788   }
 789 
 790   virtual bool is_string_late_inline() const { return true; }
 791 
 792   virtual CallGenerator* with_call_node(CallNode* call) {
 793     LateInlineStringCallGenerator* cg = new LateInlineStringCallGenerator(method(), _inline_cg);
 794     cg->set_call_node(call->as_CallStaticJava());
 795     return cg;
 796   }
 797 };
 798 
 799 CallGenerator* CallGenerator::for_string_late_inline(ciMethod* method, CallGenerator* inline_cg) {
 800   return new LateInlineStringCallGenerator(method, inline_cg);
 801 }
 802 
 803 class LateInlineBoxingCallGenerator : public LateInlineCallGenerator {
 804 
 805  public:
 806   LateInlineBoxingCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
 807     LateInlineCallGenerator(method, inline_cg, /*is_pure=*/true) {}
 808 
 809   virtual JVMState* generate(JVMState* jvms) {
 810     Compile *C = Compile::current();
 811 
 812     C->log_inline_id(this);
 813 
 814     C->add_boxing_late_inline(this);
 815 
 816     JVMState* new_jvms = DirectCallGenerator::generate(jvms);
 817     return new_jvms;
 818   }
 819 
 820   virtual bool is_boxing_late_inline() const { return true; }
 821 
 822   virtual CallGenerator* with_call_node(CallNode* call) {
 823     LateInlineBoxingCallGenerator* cg = new LateInlineBoxingCallGenerator(method(), _inline_cg);
 824     cg->set_call_node(call->as_CallStaticJava());
 825     return cg;
 826   }
 827 };
 828 
 829 CallGenerator* CallGenerator::for_boxing_late_inline(ciMethod* method, CallGenerator* inline_cg) {
 830   return new LateInlineBoxingCallGenerator(method, inline_cg);
 831 }
 832 
 833 class LateInlineVectorReboxingCallGenerator : public LateInlineCallGenerator {
 834 
 835  public:
 836   LateInlineVectorReboxingCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
 837     LateInlineCallGenerator(method, inline_cg, /*is_pure=*/true) {}
 838 
 839   virtual JVMState* generate(JVMState* jvms) {
 840     Compile *C = Compile::current();
 841 
 842     C->log_inline_id(this);
 843 
 844     C->add_vector_reboxing_late_inline(this);
 845 
 846     JVMState* new_jvms = DirectCallGenerator::generate(jvms);
 847     return new_jvms;
 848   }
 849 
 850   virtual CallGenerator* with_call_node(CallNode* call) {
 851     LateInlineVectorReboxingCallGenerator* cg = new LateInlineVectorReboxingCallGenerator(method(), _inline_cg);
 852     cg->set_call_node(call->as_CallStaticJava());
 853     return cg;
 854   }
 855 };
 856 
 857 //   static CallGenerator* for_vector_reboxing_late_inline(ciMethod* m, CallGenerator* inline_cg);
 858 CallGenerator* CallGenerator::for_vector_reboxing_late_inline(ciMethod* method, CallGenerator* inline_cg) {
 859   return new LateInlineVectorReboxingCallGenerator(method, inline_cg);
 860 }
 861 
 862 //------------------------PredictedCallGenerator------------------------------
 863 // Internal class which handles all out-of-line calls checking receiver type.
 864 class PredictedCallGenerator : public CallGenerator {
 865   ciKlass*       _predicted_receiver;
 866   CallGenerator* _if_missed;
 867   CallGenerator* _if_hit;
 868   float          _hit_prob;
 869   bool           _exact_check;
 870 
 871 public:
 872   PredictedCallGenerator(ciKlass* predicted_receiver,
 873                          CallGenerator* if_missed,
 874                          CallGenerator* if_hit, bool exact_check,
 875                          float hit_prob)
 876     : CallGenerator(if_missed->method())
 877   {
 878     // The call profile data may predict the hit_prob as extreme as 0 or 1.
 879     // Remove the extremes values from the range.
 880     if (hit_prob > PROB_MAX)   hit_prob = PROB_MAX;
 881     if (hit_prob < PROB_MIN)   hit_prob = PROB_MIN;
 882 
 883     _predicted_receiver = predicted_receiver;
 884     _if_missed          = if_missed;
 885     _if_hit             = if_hit;
 886     _hit_prob           = hit_prob;
 887     _exact_check        = exact_check;
 888   }
 889 
 890   virtual bool      is_virtual()   const    { return true; }
 891   virtual bool      is_inline()    const    { return _if_hit->is_inline(); }
 892   virtual bool      is_deferred()  const    { return _if_hit->is_deferred(); }
 893 
 894   virtual JVMState* generate(JVMState* jvms);
 895 };
 896 
 897 
 898 CallGenerator* CallGenerator::for_predicted_call(ciKlass* predicted_receiver,
 899                                                  CallGenerator* if_missed,
 900                                                  CallGenerator* if_hit,
 901                                                  float hit_prob) {
 902   return new PredictedCallGenerator(predicted_receiver, if_missed, if_hit,
 903                                     /*exact_check=*/true, hit_prob);
 904 }
 905 
 906 CallGenerator* CallGenerator::for_guarded_call(ciKlass* guarded_receiver,
 907                                                CallGenerator* if_missed,
 908                                                CallGenerator* if_hit) {
 909   return new PredictedCallGenerator(guarded_receiver, if_missed, if_hit,
 910                                     /*exact_check=*/false, PROB_ALWAYS);
 911 }
 912 
 913 JVMState* PredictedCallGenerator::generate(JVMState* jvms) {
 914   GraphKit kit(jvms);
 915   kit.C->print_inlining_update(this);
 916   PhaseGVN& gvn = kit.gvn();
 917   // We need an explicit receiver null_check before checking its type.
 918   // We share a map with the caller, so his JVMS gets adjusted.
 919   Node* receiver = kit.argument(0);
 920   CompileLog* log = kit.C->log();
 921   if (log != NULL) {
 922     log->elem("predicted_call bci='%d' exact='%d' klass='%d'",
 923               jvms->bci(), (_exact_check ? 1 : 0), log->identify(_predicted_receiver));
 924   }
 925 
 926   receiver = kit.null_check_receiver_before_call(method());
 927   if (kit.stopped()) {
 928     return kit.transfer_exceptions_into_jvms();
 929   }
 930 
 931   // Make a copy of the replaced nodes in case we need to restore them
 932   ReplacedNodes replaced_nodes = kit.map()->replaced_nodes();
 933   replaced_nodes.clone();
 934 
 935   Node* casted_receiver = receiver;  // will get updated in place...
 936   Node* slow_ctl = NULL;
 937   if (_exact_check) {
 938     slow_ctl = kit.type_check_receiver(receiver, _predicted_receiver, _hit_prob,
 939                                        &casted_receiver);
 940   } else {
 941     slow_ctl = kit.subtype_check_receiver(receiver, _predicted_receiver,
 942                                           &casted_receiver);
 943   }
 944 
 945   SafePointNode* slow_map = NULL;
 946   JVMState* slow_jvms = NULL;
 947   { PreserveJVMState pjvms(&kit);
 948     kit.set_control(slow_ctl);
 949     if (!kit.stopped()) {
 950       slow_jvms = _if_missed->generate(kit.sync_jvms());
 951       if (kit.failing())
 952         return NULL;  // might happen because of NodeCountInliningCutoff
 953       assert(slow_jvms != NULL, "must be");
 954       kit.add_exception_states_from(slow_jvms);
 955       kit.set_map(slow_jvms->map());
 956       if (!kit.stopped())
 957         slow_map = kit.stop();
 958     }
 959   }
 960 
 961   if (kit.stopped()) {
 962     // Instance does not match the predicted type.
 963     kit.set_jvms(slow_jvms);
 964     return kit.transfer_exceptions_into_jvms();
 965   }
 966 
 967   // Fall through if the instance matches the desired type.
 968   kit.replace_in_map(receiver, casted_receiver);
 969 
 970   // Make the hot call:
 971   JVMState* new_jvms = _if_hit->generate(kit.sync_jvms());
 972   if (new_jvms == NULL) {
 973     // Inline failed, so make a direct call.
 974     assert(_if_hit->is_inline(), "must have been a failed inline");
 975     CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method());
 976     new_jvms = cg->generate(kit.sync_jvms());
 977   }
 978   kit.add_exception_states_from(new_jvms);
 979   kit.set_jvms(new_jvms);
 980 
 981   // Need to merge slow and fast?
 982   if (slow_map == NULL) {
 983     // The fast path is the only path remaining.
 984     return kit.transfer_exceptions_into_jvms();
 985   }
 986 
 987   if (kit.stopped()) {
 988     // Inlined method threw an exception, so it's just the slow path after all.
 989     kit.set_jvms(slow_jvms);
 990     return kit.transfer_exceptions_into_jvms();
 991   }
 992 
 993   // There are 2 branches and the replaced nodes are only valid on
 994   // one: restore the replaced nodes to what they were before the
 995   // branch.
 996   kit.map()->set_replaced_nodes(replaced_nodes);
 997 
 998   // Finish the diamond.
 999   kit.C->set_has_split_ifs(true); // Has chance for split-if optimization
1000   RegionNode* region = new RegionNode(3);
1001   region->init_req(1, kit.control());
1002   region->init_req(2, slow_map->control());
1003   kit.set_control(gvn.transform(region));
1004   Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO);
1005   iophi->set_req(2, slow_map->i_o());
1006   kit.set_i_o(gvn.transform(iophi));
1007   // Merge memory
1008   kit.merge_memory(slow_map->merged_memory(), region, 2);
1009   // Transform new memory Phis.
1010   for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) {
1011     Node* phi = mms.memory();
1012     if (phi->is_Phi() && phi->in(0) == region) {
1013       mms.set_memory(gvn.transform(phi));
1014     }
1015   }
1016   uint tos = kit.jvms()->stkoff() + kit.sp();
1017   uint limit = slow_map->req();
1018   for (uint i = TypeFunc::Parms; i < limit; i++) {
1019     // Skip unused stack slots; fast forward to monoff();
1020     if (i == tos) {
1021       i = kit.jvms()->monoff();
1022       if( i >= limit ) break;
1023     }
1024     Node* m = kit.map()->in(i);
1025     Node* n = slow_map->in(i);
1026     if (m != n) {
1027       const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
1028       Node* phi = PhiNode::make(region, m, t);
1029       phi->set_req(2, n);
1030       kit.map()->set_req(i, gvn.transform(phi));
1031     }
1032   }
1033   return kit.transfer_exceptions_into_jvms();
1034 }
1035 
1036 
1037 CallGenerator* CallGenerator::for_method_handle_call(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline) {
1038   assert(callee->is_method_handle_intrinsic(), "for_method_handle_call mismatch");
1039   bool input_not_const;
1040   CallGenerator* cg = CallGenerator::for_method_handle_inline(jvms, caller, callee, allow_inline, input_not_const);
1041   Compile* C = Compile::current();
1042   if (cg != NULL) {
1043     if (AlwaysIncrementalInline) {
1044       return CallGenerator::for_late_inline(callee, cg);
1045     } else {
1046       return cg;
1047     }
1048   }
1049   int bci = jvms->bci();
1050   ciCallProfile profile = caller->call_profile_at_bci(bci);
1051   int call_site_count = caller->scale_count(profile.count());
1052 
1053   if (IncrementalInlineMH && call_site_count > 0 &&
1054       (input_not_const || !C->inlining_incrementally() || C->over_inlining_cutoff())) {
1055     return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
1056   } else {
1057     // Out-of-line call.
1058     return CallGenerator::for_direct_call(callee);
1059   }
1060 }
1061 
1062 class NativeCallGenerator : public CallGenerator {
1063 private:
1064   address _call_addr;
1065   ciNativeEntryPoint* _nep;
1066 public:
1067   NativeCallGenerator(ciMethod* m, address call_addr, ciNativeEntryPoint* nep)
1068    : CallGenerator(m), _call_addr(call_addr), _nep(nep) {}
1069 
1070   virtual JVMState* generate(JVMState* jvms);
1071 };
1072 
1073 JVMState* NativeCallGenerator::generate(JVMState* jvms) {
1074   GraphKit kit(jvms);
1075 
1076   Node* call = kit.make_native_call(_call_addr, tf(), method()->arg_size(), _nep); // -fallback, - nep
1077   if (call == NULL) return NULL;
1078 
1079   kit.C->print_inlining_update(this);
1080   if (kit.C->log() != NULL) {
1081     kit.C->log()->elem("l2n_intrinsification_success bci='%d' entry_point='" INTPTR_FORMAT "'", jvms->bci(), p2i(_call_addr));
1082   }
1083 
1084   return kit.transfer_exceptions_into_jvms();
1085 }
1086 
1087 CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline, bool& input_not_const) {
1088   GraphKit kit(jvms);
1089   PhaseGVN& gvn = kit.gvn();
1090   Compile* C = kit.C;
1091   vmIntrinsics::ID iid = callee->intrinsic_id();
1092   input_not_const = true;
1093   if (StressMethodHandleLinkerInlining) {
1094     allow_inline = false;
1095   }
1096   switch (iid) {
1097   case vmIntrinsics::_invokeBasic:
1098     {
1099       // Get MethodHandle receiver:
1100       Node* receiver = kit.argument(0);
1101       if (receiver->Opcode() == Op_ConP) {
1102         input_not_const = false;
1103         const TypeOopPtr* oop_ptr = receiver->bottom_type()->is_oopptr();
1104         ciMethod* target = oop_ptr->const_oop()->as_method_handle()->get_vmtarget();
1105         const int vtable_index = Method::invalid_vtable_index;
1106 
1107         if (!ciMethod::is_consistent_info(callee, target)) {
1108           print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1109                                  "signatures mismatch");
1110           return NULL;
1111         }
1112 
1113         CallGenerator* cg = C->call_generator(target, vtable_index,
1114                                               false /* call_does_dispatch */,
1115                                               jvms,
1116                                               allow_inline,
1117                                               PROB_ALWAYS);
1118         return cg;
1119       } else {
1120         print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1121                                "receiver not constant");
1122       }
1123     }
1124     break;
1125 
1126   case vmIntrinsics::_linkToVirtual:
1127   case vmIntrinsics::_linkToStatic:
1128   case vmIntrinsics::_linkToSpecial:
1129   case vmIntrinsics::_linkToInterface:
1130     {
1131       // Get MemberName argument:
1132       Node* member_name = kit.argument(callee->arg_size() - 1);
1133       if (member_name->Opcode() == Op_ConP) {
1134         input_not_const = false;
1135         const TypeOopPtr* oop_ptr = member_name->bottom_type()->is_oopptr();
1136         ciMethod* target = oop_ptr->const_oop()->as_member_name()->get_vmtarget();
1137 
1138         if (!ciMethod::is_consistent_info(callee, target)) {
1139           print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1140                                  "signatures mismatch");
1141           return NULL;
1142         }
1143 
1144         // In lambda forms we erase signature types to avoid resolving issues
1145         // involving class loaders.  When we optimize a method handle invoke
1146         // to a direct call we must cast the receiver and arguments to its
1147         // actual types.
1148         ciSignature* signature = target->signature();
1149         const int receiver_skip = target->is_static() ? 0 : 1;
1150         // Cast receiver to its type.
1151         if (!target->is_static()) {
1152           Node* arg = kit.argument(0);
1153           const TypeOopPtr* arg_type = arg->bottom_type()->isa_oopptr();
1154           const Type*       sig_type = TypeOopPtr::make_from_klass(signature->accessing_klass());
1155           if (arg_type != NULL && !arg_type->higher_equal(sig_type)) {
1156             const Type* recv_type = arg_type->filter_speculative(sig_type); // keep speculative part
1157             Node* cast_obj = gvn.transform(new CheckCastPPNode(kit.control(), arg, recv_type));
1158             kit.set_argument(0, cast_obj);
1159           }
1160         }
1161         // Cast reference arguments to its type.
1162         for (int i = 0, j = 0; i < signature->count(); i++) {
1163           ciType* t = signature->type_at(i);
1164           if (t->is_klass()) {
1165             Node* arg = kit.argument(receiver_skip + j);
1166             const TypeOopPtr* arg_type = arg->bottom_type()->isa_oopptr();
1167             const Type*       sig_type = TypeOopPtr::make_from_klass(t->as_klass());
1168             if (arg_type != NULL && !arg_type->higher_equal(sig_type)) {
1169               const Type* narrowed_arg_type = arg_type->filter_speculative(sig_type); // keep speculative part
1170               Node* cast_obj = gvn.transform(new CheckCastPPNode(kit.control(), arg, narrowed_arg_type));
1171               kit.set_argument(receiver_skip + j, cast_obj);
1172             }
1173           }
1174           j += t->size();  // long and double take two slots
1175         }
1176 
1177         // Try to get the most accurate receiver type
1178         const bool is_virtual              = (iid == vmIntrinsics::_linkToVirtual);
1179         const bool is_virtual_or_interface = (is_virtual || iid == vmIntrinsics::_linkToInterface);
1180         int  vtable_index       = Method::invalid_vtable_index;
1181         bool call_does_dispatch = false;
1182 
1183         ciKlass* speculative_receiver_type = NULL;
1184         if (is_virtual_or_interface) {
1185           ciInstanceKlass* klass = target->holder();
1186           Node*             receiver_node = kit.argument(0);
1187           const TypeOopPtr* receiver_type = gvn.type(receiver_node)->isa_oopptr();
1188           // call_does_dispatch and vtable_index are out-parameters.  They might be changed.
1189           // optimize_virtual_call() takes 2 different holder
1190           // arguments for a corner case that doesn't apply here (see
1191           // Parse::do_call())
1192           target = C->optimize_virtual_call(caller, klass, klass,
1193                                             target, receiver_type, is_virtual,
1194                                             call_does_dispatch, vtable_index, // out-parameters
1195                                             false /* check_access */);
1196           // We lack profiling at this call but type speculation may
1197           // provide us with a type
1198           speculative_receiver_type = (receiver_type != NULL) ? receiver_type->speculative_type() : NULL;
1199         }
1200         CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms,
1201                                               allow_inline,
1202                                               PROB_ALWAYS,
1203                                               speculative_receiver_type);
1204         return cg;
1205       } else {
1206         print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1207                                "member_name not constant");
1208       }
1209     }
1210     break;
1211 
1212     case vmIntrinsics::_linkToNative:
1213     {
1214       Node* addr_n = kit.argument(1); // target address
1215       Node* nep_n = kit.argument(callee->arg_size() - 1); // NativeEntryPoint
1216       // This check needs to be kept in sync with the one in CallStaticJavaNode::Ideal
1217       if (addr_n->Opcode() == Op_ConL && nep_n->Opcode() == Op_ConP) {
1218         input_not_const = false;
1219         const TypeLong* addr_t = addr_n->bottom_type()->is_long();
1220         const TypeOopPtr* nep_t = nep_n->bottom_type()->is_oopptr();
1221         address addr = (address) addr_t->get_con();
1222         ciNativeEntryPoint* nep = nep_t->const_oop()->as_native_entry_point();
1223         return new NativeCallGenerator(callee, addr, nep);
1224       } else {
1225         print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1226                                "NativeEntryPoint not constant");
1227       }
1228     }
1229     break;
1230 
1231   default:
1232     fatal("unexpected intrinsic %d: %s", vmIntrinsics::as_int(iid), vmIntrinsics::name_at(iid));
1233     break;
1234   }
1235   return NULL;
1236 }
1237 
1238 
1239 //------------------------PredicatedIntrinsicGenerator------------------------------
1240 // Internal class which handles all predicated Intrinsic calls.
1241 class PredicatedIntrinsicGenerator : public CallGenerator {
1242   CallGenerator* _intrinsic;
1243   CallGenerator* _cg;
1244 
1245 public:
1246   PredicatedIntrinsicGenerator(CallGenerator* intrinsic,
1247                                CallGenerator* cg)
1248     : CallGenerator(cg->method())
1249   {
1250     _intrinsic = intrinsic;
1251     _cg        = cg;
1252   }
1253 
1254   virtual bool      is_virtual()   const    { return true; }
1255   virtual bool      is_inline()    const    { return true; }
1256   virtual bool      is_intrinsic() const    { return true; }
1257 
1258   virtual JVMState* generate(JVMState* jvms);
1259 };
1260 
1261 
1262 CallGenerator* CallGenerator::for_predicated_intrinsic(CallGenerator* intrinsic,
1263                                                        CallGenerator* cg) {
1264   return new PredicatedIntrinsicGenerator(intrinsic, cg);
1265 }
1266 
1267 
1268 JVMState* PredicatedIntrinsicGenerator::generate(JVMState* jvms) {
1269   // The code we want to generate here is:
1270   //    if (receiver == NULL)
1271   //        uncommon_Trap
1272   //    if (predicate(0))
1273   //        do_intrinsic(0)
1274   //    else
1275   //    if (predicate(1))
1276   //        do_intrinsic(1)
1277   //    ...
1278   //    else
1279   //        do_java_comp
1280 
1281   GraphKit kit(jvms);
1282   PhaseGVN& gvn = kit.gvn();
1283 
1284   CompileLog* log = kit.C->log();
1285   if (log != NULL) {
1286     log->elem("predicated_intrinsic bci='%d' method='%d'",
1287               jvms->bci(), log->identify(method()));
1288   }
1289 
1290   if (!method()->is_static()) {
1291     // We need an explicit receiver null_check before checking its type in predicate.
1292     // We share a map with the caller, so his JVMS gets adjusted.
1293     Node* receiver = kit.null_check_receiver_before_call(method());
1294     if (kit.stopped()) {
1295       return kit.transfer_exceptions_into_jvms();
1296     }
1297   }
1298 
1299   int n_predicates = _intrinsic->predicates_count();
1300   assert(n_predicates > 0, "sanity");
1301 
1302   JVMState** result_jvms = NEW_RESOURCE_ARRAY(JVMState*, (n_predicates+1));
1303 
1304   // Region for normal compilation code if intrinsic failed.
1305   Node* slow_region = new RegionNode(1);
1306 
1307   int results = 0;
1308   for (int predicate = 0; (predicate < n_predicates) && !kit.stopped(); predicate++) {
1309 #ifdef ASSERT
1310     JVMState* old_jvms = kit.jvms();
1311     SafePointNode* old_map = kit.map();
1312     Node* old_io  = old_map->i_o();
1313     Node* old_mem = old_map->memory();
1314     Node* old_exc = old_map->next_exception();
1315 #endif
1316     Node* else_ctrl = _intrinsic->generate_predicate(kit.sync_jvms(), predicate);
1317 #ifdef ASSERT
1318     // Assert(no_new_memory && no_new_io && no_new_exceptions) after generate_predicate.
1319     assert(old_jvms == kit.jvms(), "generate_predicate should not change jvm state");
1320     SafePointNode* new_map = kit.map();
1321     assert(old_io  == new_map->i_o(), "generate_predicate should not change i_o");
1322     assert(old_mem == new_map->memory(), "generate_predicate should not change memory");
1323     assert(old_exc == new_map->next_exception(), "generate_predicate should not add exceptions");
1324 #endif
1325     if (!kit.stopped()) {
1326       PreserveJVMState pjvms(&kit);
1327       // Generate intrinsic code:
1328       JVMState* new_jvms = _intrinsic->generate(kit.sync_jvms());
1329       if (new_jvms == NULL) {
1330         // Intrinsic failed, use normal compilation path for this predicate.
1331         slow_region->add_req(kit.control());
1332       } else {
1333         kit.add_exception_states_from(new_jvms);
1334         kit.set_jvms(new_jvms);
1335         if (!kit.stopped()) {
1336           result_jvms[results++] = kit.jvms();
1337         }
1338       }
1339     }
1340     if (else_ctrl == NULL) {
1341       else_ctrl = kit.C->top();
1342     }
1343     kit.set_control(else_ctrl);
1344   }
1345   if (!kit.stopped()) {
1346     // Final 'else' after predicates.
1347     slow_region->add_req(kit.control());
1348   }
1349   if (slow_region->req() > 1) {
1350     PreserveJVMState pjvms(&kit);
1351     // Generate normal compilation code:
1352     kit.set_control(gvn.transform(slow_region));
1353     JVMState* new_jvms = _cg->generate(kit.sync_jvms());
1354     if (kit.failing())
1355       return NULL;  // might happen because of NodeCountInliningCutoff
1356     assert(new_jvms != NULL, "must be");
1357     kit.add_exception_states_from(new_jvms);
1358     kit.set_jvms(new_jvms);
1359     if (!kit.stopped()) {
1360       result_jvms[results++] = kit.jvms();
1361     }
1362   }
1363 
1364   if (results == 0) {
1365     // All paths ended in uncommon traps.
1366     (void) kit.stop();
1367     return kit.transfer_exceptions_into_jvms();
1368   }
1369 
1370   if (results == 1) { // Only one path
1371     kit.set_jvms(result_jvms[0]);
1372     return kit.transfer_exceptions_into_jvms();
1373   }
1374 
1375   // Merge all paths.
1376   kit.C->set_has_split_ifs(true); // Has chance for split-if optimization
1377   RegionNode* region = new RegionNode(results + 1);
1378   Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO);
1379   for (int i = 0; i < results; i++) {
1380     JVMState* jvms = result_jvms[i];
1381     int path = i + 1;
1382     SafePointNode* map = jvms->map();
1383     region->init_req(path, map->control());
1384     iophi->set_req(path, map->i_o());
1385     if (i == 0) {
1386       kit.set_jvms(jvms);
1387     } else {
1388       kit.merge_memory(map->merged_memory(), region, path);
1389     }
1390   }
1391   kit.set_control(gvn.transform(region));
1392   kit.set_i_o(gvn.transform(iophi));
1393   // Transform new memory Phis.
1394   for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) {
1395     Node* phi = mms.memory();
1396     if (phi->is_Phi() && phi->in(0) == region) {
1397       mms.set_memory(gvn.transform(phi));
1398     }
1399   }
1400 
1401   // Merge debug info.
1402   Node** ins = NEW_RESOURCE_ARRAY(Node*, results);
1403   uint tos = kit.jvms()->stkoff() + kit.sp();
1404   Node* map = kit.map();
1405   uint limit = map->req();
1406   for (uint i = TypeFunc::Parms; i < limit; i++) {
1407     // Skip unused stack slots; fast forward to monoff();
1408     if (i == tos) {
1409       i = kit.jvms()->monoff();
1410       if( i >= limit ) break;
1411     }
1412     Node* n = map->in(i);
1413     ins[0] = n;
1414     const Type* t = gvn.type(n);
1415     bool needs_phi = false;
1416     for (int j = 1; j < results; j++) {
1417       JVMState* jvms = result_jvms[j];
1418       Node* jmap = jvms->map();
1419       Node* m = NULL;
1420       if (jmap->req() > i) {
1421         m = jmap->in(i);
1422         if (m != n) {
1423           needs_phi = true;
1424           t = t->meet_speculative(gvn.type(m));
1425         }
1426       }
1427       ins[j] = m;
1428     }
1429     if (needs_phi) {
1430       Node* phi = PhiNode::make(region, n, t);
1431       for (int j = 1; j < results; j++) {
1432         phi->set_req(j + 1, ins[j]);
1433       }
1434       map->set_req(i, gvn.transform(phi));
1435     }
1436   }
1437 
1438   return kit.transfer_exceptions_into_jvms();
1439 }
1440 
1441 //-------------------------UncommonTrapCallGenerator-----------------------------
1442 // Internal class which handles all out-of-line calls checking receiver type.
1443 class UncommonTrapCallGenerator : public CallGenerator {
1444   Deoptimization::DeoptReason _reason;
1445   Deoptimization::DeoptAction _action;
1446 
1447 public:
1448   UncommonTrapCallGenerator(ciMethod* m,
1449                             Deoptimization::DeoptReason reason,
1450                             Deoptimization::DeoptAction action)
1451     : CallGenerator(m)
1452   {
1453     _reason = reason;
1454     _action = action;
1455   }
1456 
1457   virtual bool      is_virtual() const          { ShouldNotReachHere(); return false; }
1458   virtual bool      is_trap() const             { return true; }
1459 
1460   virtual JVMState* generate(JVMState* jvms);
1461 };
1462 
1463 
1464 CallGenerator*
1465 CallGenerator::for_uncommon_trap(ciMethod* m,
1466                                  Deoptimization::DeoptReason reason,
1467                                  Deoptimization::DeoptAction action) {
1468   return new UncommonTrapCallGenerator(m, reason, action);
1469 }
1470 
1471 
1472 JVMState* UncommonTrapCallGenerator::generate(JVMState* jvms) {
1473   GraphKit kit(jvms);
1474   kit.C->print_inlining_update(this);
1475   // Take the trap with arguments pushed on the stack.  (Cf. null_check_receiver).
1476   // Callsite signature can be different from actual method being called (i.e _linkTo* sites).
1477   // Use callsite signature always.
1478   ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci());
1479   int nargs = declared_method->arg_size();
1480   kit.inc_sp(nargs);
1481   assert(nargs <= kit.sp() && kit.sp() <= jvms->stk_size(), "sane sp w/ args pushed");
1482   if (_reason == Deoptimization::Reason_class_check &&
1483       _action == Deoptimization::Action_maybe_recompile) {
1484     // Temp fix for 6529811
1485     // Don't allow uncommon_trap to override our decision to recompile in the event
1486     // of a class cast failure for a monomorphic call as it will never let us convert
1487     // the call to either bi-morphic or megamorphic and can lead to unc-trap loops
1488     bool keep_exact_action = true;
1489     kit.uncommon_trap(_reason, _action, NULL, "monomorphic vcall checkcast", false, keep_exact_action);
1490   } else {
1491     kit.uncommon_trap(_reason, _action);
1492   }
1493   return kit.transfer_exceptions_into_jvms();
1494 }
1495 
1496 // (Note:  Moved hook_up_call to GraphKit::set_edges_for_java_call.)
1497 
1498 // (Node:  Merged hook_up_exits into ParseGenerator::generate.)