1 /*
   2  * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "ci/bcEscapeAnalyzer.hpp"
  27 #include "ci/ciCallSite.hpp"
  28 #include "ci/ciObjArray.hpp"
  29 #include "ci/ciMemberName.hpp"
  30 #include "ci/ciMethodHandle.hpp"
  31 #include "classfile/javaClasses.hpp"
  32 #include "compiler/compileLog.hpp"
  33 #include "opto/addnode.hpp"
  34 #include "opto/callGenerator.hpp"
  35 #include "opto/callnode.hpp"
  36 #include "opto/castnode.hpp"
  37 #include "opto/cfgnode.hpp"
  38 #include "opto/parse.hpp"
  39 #include "opto/rootnode.hpp"
  40 #include "opto/runtime.hpp"
  41 #include "opto/subnode.hpp"
  42 #include "opto/valuetypenode.hpp"
  43 #include "runtime/sharedRuntime.hpp"
  44 
  45 // Utility function.
  46 const TypeFunc* CallGenerator::tf() const {
  47   return TypeFunc::make(method());
  48 }
  49 
  50 bool CallGenerator::is_inlined_method_handle_intrinsic(JVMState* jvms, ciMethod* m) {
  51   return is_inlined_method_handle_intrinsic(jvms->method(), jvms->bci(), m);
  52 }
  53 
  54 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* caller, int bci, ciMethod* m) {
  55   ciMethod* symbolic_info = caller->get_method_at_bci(bci);
  56   return is_inlined_method_handle_intrinsic(symbolic_info, m);
  57 }
  58 
  59 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* symbolic_info, ciMethod* m) {
  60   return symbolic_info->is_method_handle_intrinsic() && !m->is_method_handle_intrinsic();
  61 }
  62 
  63 //-----------------------------ParseGenerator---------------------------------
  64 // Internal class which handles all direct bytecode traversal.
  65 class ParseGenerator : public InlineCallGenerator {
  66 private:
  67   bool  _is_osr;
  68   float _expected_uses;
  69 
  70 public:
  71   ParseGenerator(ciMethod* method, float expected_uses, bool is_osr = false)
  72     : InlineCallGenerator(method)
  73   {
  74     _is_osr        = is_osr;
  75     _expected_uses = expected_uses;
  76     assert(InlineTree::check_can_parse(method) == NULL, "parse must be possible");
  77   }
  78 
  79   virtual bool      is_parse() const           { return true; }
  80   virtual JVMState* generate(JVMState* jvms);
  81   int is_osr() { return _is_osr; }
  82 
  83 };
  84 
  85 JVMState* ParseGenerator::generate(JVMState* jvms) {
  86   Compile* C = Compile::current();
  87   C->print_inlining_update(this);
  88 
  89   if (is_osr()) {
  90     // The JVMS for a OSR has a single argument (see its TypeFunc).
  91     assert(jvms->depth() == 1, "no inline OSR");
  92   }
  93 
  94   if (C->failing()) {
  95     return NULL;  // bailing out of the compile; do not try to parse
  96   }
  97 
  98   Parse parser(jvms, method(), _expected_uses);
  99   // Grab signature for matching/allocation
 100 #ifdef ASSERT
 101   if (parser.tf() != (parser.depth() == 1 ? C->tf() : tf())) {
 102     assert(C->env()->system_dictionary_modification_counter_changed(),
 103            "Must invalidate if TypeFuncs differ");
 104   }
 105 #endif
 106 
 107   GraphKit& exits = parser.exits();
 108 
 109   if (C->failing()) {
 110     while (exits.pop_exception_state() != NULL) ;
 111     return NULL;
 112   }
 113 
 114   assert(exits.jvms()->same_calls_as(jvms), "sanity");
 115 
 116   // Simply return the exit state of the parser,
 117   // augmented by any exceptional states.
 118   return exits.transfer_exceptions_into_jvms();
 119 }
 120 
 121 //---------------------------DirectCallGenerator------------------------------
 122 // Internal class which handles all out-of-line calls w/o receiver type checks.
 123 class DirectCallGenerator : public CallGenerator {
 124  private:
 125   CallStaticJavaNode* _call_node;
 126   // Force separate memory and I/O projections for the exceptional
 127   // paths to facilitate late inlining.
 128   bool                _separate_io_proj;
 129 
 130  public:
 131   DirectCallGenerator(ciMethod* method, bool separate_io_proj)
 132     : CallGenerator(method),
 133       _call_node(NULL),
 134       _separate_io_proj(separate_io_proj)
 135   {
 136     if (ValueTypeReturnedAsFields && method->is_method_handle_intrinsic()) {
 137       // If that call has not been optimized by the time optimizations are over,
 138       // we'll need to add a call to create a value type instance from the klass
 139       // returned by the call (see PhaseMacroExpand::expand_mh_intrinsic_return).
 140       // Separating memory and I/O projections for exceptions is required to
 141       // perform that graph transformation.
 142       _separate_io_proj = true;
 143     }
 144   }
 145   virtual JVMState* generate(JVMState* jvms);
 146 
 147   CallStaticJavaNode* call_node() const { return _call_node; }
 148 };
 149 
 150 JVMState* DirectCallGenerator::generate(JVMState* jvms) {
 151   GraphKit kit(jvms);
 152   kit.C->print_inlining_update(this);
 153   PhaseGVN& gvn = kit.gvn();
 154   bool is_static = method()->is_static();
 155   address target = is_static ? SharedRuntime::get_resolve_static_call_stub()
 156                              : SharedRuntime::get_resolve_opt_virtual_call_stub();
 157 
 158   if (kit.C->log() != NULL) {
 159     kit.C->log()->elem("direct_call bci='%d'", jvms->bci());
 160   }
 161 
 162   CallStaticJavaNode *call = new CallStaticJavaNode(kit.C, tf(), target, method(), kit.bci());
 163   if (is_inlined_method_handle_intrinsic(jvms, method())) {
 164     // To be able to issue a direct call and skip a call to MH.linkTo*/invokeBasic adapter,
 165     // additional information about the method being invoked should be attached
 166     // to the call site to make resolution logic work
 167     // (see SharedRuntime::resolve_static_call_C).
 168     call->set_override_symbolic_info(true);
 169   }
 170   _call_node = call;  // Save the call node in case we need it later
 171   if (!is_static) {
 172     // Make an explicit receiver null_check as part of this call.
 173     // Since we share a map with the caller, his JVMS gets adjusted.
 174     kit.null_check_receiver_before_call(method());
 175     if (kit.stopped()) {
 176       // And dump it back to the caller, decorated with any exceptions:
 177       return kit.transfer_exceptions_into_jvms();
 178     }
 179     // Mark the call node as virtual, sort of:
 180     call->set_optimized_virtual(true);
 181     if (method()->is_method_handle_intrinsic() ||
 182         method()->is_compiled_lambda_form()) {
 183       call->set_method_handle_invoke(true);
 184     }
 185   }
 186   kit.set_arguments_for_java_call(call, is_late_inline());
 187   if (kit.stopped()) {
 188     return kit.transfer_exceptions_into_jvms();
 189   }
 190   kit.set_edges_for_java_call(call, false, _separate_io_proj);
 191   Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
 192   kit.push_node(method()->return_type()->basic_type(), ret);
 193   return kit.transfer_exceptions_into_jvms();
 194 }
 195 
 196 //--------------------------VirtualCallGenerator------------------------------
 197 // Internal class which handles all out-of-line calls checking receiver type.
 198 class VirtualCallGenerator : public CallGenerator {
 199 private:
 200   int _vtable_index;
 201 public:
 202   VirtualCallGenerator(ciMethod* method, int vtable_index)
 203     : CallGenerator(method), _vtable_index(vtable_index)
 204   {
 205     assert(vtable_index == Method::invalid_vtable_index ||
 206            vtable_index >= 0, "either invalid or usable");
 207   }
 208   virtual bool      is_virtual() const          { return true; }
 209   virtual JVMState* generate(JVMState* jvms);
 210 };
 211 
 212 JVMState* VirtualCallGenerator::generate(JVMState* jvms) {
 213   GraphKit kit(jvms);
 214   Node* receiver = kit.argument(0);
 215   kit.C->print_inlining_update(this);
 216 
 217   if (kit.C->log() != NULL) {
 218     kit.C->log()->elem("virtual_call bci='%d'", jvms->bci());
 219   }
 220 
 221   // If the receiver is a constant null, do not torture the system
 222   // by attempting to call through it.  The compile will proceed
 223   // correctly, but may bail out in final_graph_reshaping, because
 224   // the call instruction will have a seemingly deficient out-count.
 225   // (The bailout says something misleading about an "infinite loop".)
 226   if (!receiver->is_ValueType() && kit.gvn().type(receiver)->higher_equal(TypePtr::NULL_PTR)) {
 227     assert(Bytecodes::is_invoke(kit.java_bc()), "%d: %s", kit.java_bc(), Bytecodes::name(kit.java_bc()));
 228     ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci());
 229     int arg_size = declared_method->signature()->arg_size_for_bc(kit.java_bc());
 230     kit.inc_sp(arg_size);  // restore arguments
 231     kit.uncommon_trap(Deoptimization::Reason_null_check,
 232                       Deoptimization::Action_none,
 233                       NULL, "null receiver");
 234     return kit.transfer_exceptions_into_jvms();
 235   }
 236 
 237   // Ideally we would unconditionally do a null check here and let it
 238   // be converted to an implicit check based on profile information.
 239   // However currently the conversion to implicit null checks in
 240   // Block::implicit_null_check() only looks for loads and stores, not calls.
 241   ciMethod *caller = kit.method();
 242   ciMethodData *caller_md = (caller == NULL) ? NULL : caller->method_data();
 243   if (!UseInlineCaches || !ImplicitNullChecks || !os::zero_page_read_protected() ||
 244        ((ImplicitNullCheckThreshold > 0) && caller_md &&
 245        (caller_md->trap_count(Deoptimization::Reason_null_check)
 246        >= (uint)ImplicitNullCheckThreshold))) {
 247     // Make an explicit receiver null_check as part of this call.
 248     // Since we share a map with the caller, his JVMS gets adjusted.
 249     receiver = kit.null_check_receiver_before_call(method());
 250     if (kit.stopped()) {
 251       // And dump it back to the caller, decorated with any exceptions:
 252       return kit.transfer_exceptions_into_jvms();
 253     }
 254   }
 255 
 256   assert(!method()->is_static(), "virtual call must not be to static");
 257   assert(!method()->is_final(), "virtual call should not be to final");
 258   assert(!method()->is_private(), "virtual call should not be to private");
 259   assert(_vtable_index == Method::invalid_vtable_index || !UseInlineCaches,
 260          "no vtable calls if +UseInlineCaches ");
 261   address target = SharedRuntime::get_resolve_virtual_call_stub();
 262   // Normal inline cache used for call
 263   CallDynamicJavaNode *call = new CallDynamicJavaNode(tf(), target, method(), _vtable_index, kit.bci());
 264   if (is_inlined_method_handle_intrinsic(jvms, method())) {
 265     // To be able to issue a direct call (optimized virtual or virtual)
 266     // and skip a call to MH.linkTo*/invokeBasic adapter, additional information
 267     // about the method being invoked should be attached to the call site to
 268     // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C).
 269     call->set_override_symbolic_info(true);
 270   }
 271   kit.set_arguments_for_java_call(call);
 272   if (kit.stopped()) {
 273     return kit.transfer_exceptions_into_jvms();
 274   }
 275   kit.set_edges_for_java_call(call);
 276   Node* ret = kit.set_results_for_java_call(call);
 277   kit.push_node(method()->return_type()->basic_type(), ret);
 278 
 279   // Represent the effect of an implicit receiver null_check
 280   // as part of this call.  Since we share a map with the caller,
 281   // his JVMS gets adjusted.
 282   kit.cast_not_null(receiver);
 283   return kit.transfer_exceptions_into_jvms();
 284 }
 285 
 286 CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) {
 287   if (InlineTree::check_can_parse(m) != NULL)  return NULL;
 288   return new ParseGenerator(m, expected_uses);
 289 }
 290 
 291 // As a special case, the JVMS passed to this CallGenerator is
 292 // for the method execution already in progress, not just the JVMS
 293 // of the caller.  Thus, this CallGenerator cannot be mixed with others!
 294 CallGenerator* CallGenerator::for_osr(ciMethod* m, int osr_bci) {
 295   if (InlineTree::check_can_parse(m) != NULL)  return NULL;
 296   float past_uses = m->interpreter_invocation_count();
 297   float expected_uses = past_uses;
 298   return new ParseGenerator(m, expected_uses, true);
 299 }
 300 
 301 CallGenerator* CallGenerator::for_direct_call(ciMethod* m, bool separate_io_proj) {
 302   assert(!m->is_abstract(), "for_direct_call mismatch");
 303   return new DirectCallGenerator(m, separate_io_proj);
 304 }
 305 
 306 CallGenerator* CallGenerator::for_virtual_call(ciMethod* m, int vtable_index) {
 307   assert(!m->is_static(), "for_virtual_call mismatch");
 308   assert(!m->is_method_handle_intrinsic(), "should be a direct call");
 309   return new VirtualCallGenerator(m, vtable_index);
 310 }
 311 
 312 // Allow inlining decisions to be delayed
 313 class LateInlineCallGenerator : public DirectCallGenerator {
 314  private:
 315   jlong _unique_id;   // unique id for log compilation
 316   bool _is_pure_call; // a hint that the call doesn't have important side effects to care about
 317 
 318  protected:
 319   CallGenerator* _inline_cg;
 320   virtual bool do_late_inline_check(JVMState* jvms) { return true; }
 321 
 322  public:
 323   LateInlineCallGenerator(ciMethod* method, CallGenerator* inline_cg, bool is_pure_call = false) :
 324     DirectCallGenerator(method, true), _unique_id(0), _is_pure_call(is_pure_call), _inline_cg(inline_cg) {}
 325 
 326   virtual bool is_late_inline() const { return true; }
 327 
 328   // Convert the CallStaticJava into an inline
 329   virtual void do_late_inline();
 330 
 331   virtual JVMState* generate(JVMState* jvms) {
 332     Compile *C = Compile::current();
 333 
 334     C->log_inline_id(this);
 335 
 336     // Record that this call site should be revisited once the main
 337     // parse is finished.
 338     if (!is_mh_late_inline()) {
 339       C->add_late_inline(this);
 340     }
 341 
 342     // Emit the CallStaticJava and request separate projections so
 343     // that the late inlining logic can distinguish between fall
 344     // through and exceptional uses of the memory and io projections
 345     // as is done for allocations and macro expansion.
 346     return DirectCallGenerator::generate(jvms);
 347   }
 348 
 349   virtual void print_inlining_late(const char* msg) {
 350     CallNode* call = call_node();
 351     Compile* C = Compile::current();
 352     C->print_inlining_assert_ready();
 353     C->print_inlining(method(), call->jvms()->depth()-1, call->jvms()->bci(), msg);
 354     C->print_inlining_move_to(this);
 355     C->print_inlining_update_delayed(this);
 356   }
 357 
 358   virtual void set_unique_id(jlong id) {
 359     _unique_id = id;
 360   }
 361 
 362   virtual jlong unique_id() const {
 363     return _unique_id;
 364   }
 365 };
 366 
 367 void LateInlineCallGenerator::do_late_inline() {
 368   // Can't inline it
 369   CallStaticJavaNode* call = call_node();
 370   if (call == NULL || call->outcnt() == 0 ||
 371       call->in(0) == NULL || call->in(0)->is_top()) {
 372     return;
 373   }
 374 
 375   const GrowableArray<SigEntry>* sig_cc = method()->get_sig_cc();
 376   const TypeTuple* r = call->tf()->domain_cc();
 377   for (uint i1 = TypeFunc::Parms, i2 = 0; i1 < r->cnt(); i1++) {
 378     if (sig_cc != NULL) {
 379       // Skip reserved entries
 380       while (!SigEntry::skip_value_delimiters(sig_cc, i2)) {
 381         i2++;
 382       }
 383       if (SigEntry::is_reserved_entry(sig_cc, i2++)) {
 384         assert(call->in(i1)->is_top(), "should be top");
 385         continue;
 386       }
 387     }
 388     if (call->in(i1)->is_top() && r->field_at(i1) != Type::HALF) {
 389       assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
 390       return;
 391     }
 392   }
 393 
 394   if (call->in(TypeFunc::Memory)->is_top()) {
 395     assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
 396     return;
 397   }
 398 
 399   // check for unreachable loop
 400   CallProjections* callprojs = call->extract_projections(true);
 401   if (callprojs->fallthrough_catchproj == call->in(0) ||
 402       callprojs->catchall_catchproj == call->in(0) ||
 403       callprojs->fallthrough_memproj == call->in(TypeFunc::Memory) ||
 404       callprojs->catchall_memproj == call->in(TypeFunc::Memory) ||
 405       callprojs->fallthrough_ioproj == call->in(TypeFunc::I_O) ||
 406       callprojs->catchall_ioproj == call->in(TypeFunc::I_O) ||
 407       (callprojs->exobj != NULL && call->find_edge(callprojs->exobj) != -1)) {
 408     return;
 409   }
 410   bool result_not_used = true;
 411   for (uint i = 0; i < callprojs->nb_resproj; i++) {
 412     if (callprojs->resproj[i] != NULL) {
 413       if (callprojs->resproj[i]->outcnt() != 0) {
 414         result_not_used = false;
 415       }
 416       if (call->find_edge(callprojs->resproj[i]) != -1) {
 417         return;
 418       }
 419     }
 420   }
 421 
 422   Compile* C = Compile::current();
 423   // Remove inlined methods from Compiler's lists.
 424   if (call->is_macro()) {
 425     C->remove_macro_node(call);
 426   }
 427 
 428   if (_is_pure_call && result_not_used) {
 429     // The call is marked as pure (no important side effects), but result isn't used.
 430     // It's safe to remove the call.
 431     GraphKit kit(call->jvms());
 432     kit.replace_call(call, C->top(), true);
 433   } else {
 434     // Make a clone of the JVMState that appropriate to use for driving a parse
 435     JVMState* old_jvms = call->jvms();
 436     JVMState* jvms = old_jvms->clone_shallow(C);
 437     uint size = call->req();
 438     SafePointNode* map = new SafePointNode(size, jvms);
 439     for (uint i1 = 0; i1 < size; i1++) {
 440       map->init_req(i1, call->in(i1));
 441     }
 442 
 443     PhaseGVN& gvn = *C->initial_gvn();
 444     // Make sure the state is a MergeMem for parsing.
 445     if (!map->in(TypeFunc::Memory)->is_MergeMem()) {
 446       Node* mem = MergeMemNode::make(map->in(TypeFunc::Memory));
 447       gvn.set_type_bottom(mem);
 448       map->set_req(TypeFunc::Memory, mem);
 449     }
 450 
 451     // blow away old call arguments
 452     for (uint i1 = TypeFunc::Parms; i1 < r->cnt(); i1++) {
 453       map->set_req(i1, C->top());
 454     }
 455     jvms->set_map(map);
 456 
 457     // Make enough space in the expression stack to transfer
 458     // the incoming arguments and return value.
 459     map->ensure_stack(jvms, jvms->method()->max_stack());
 460     const TypeTuple *domain_sig = call->_tf->domain_sig();
 461     ExtendedSignature sig_cc = ExtendedSignature(method()->get_sig_cc(), SigEntryFilter());
 462     uint nargs = method()->arg_size();
 463     assert(domain_sig->cnt() - TypeFunc::Parms == nargs, "inconsistent signature");
 464 
 465     uint j = TypeFunc::Parms;
 466     for (uint i1 = 0; i1 < nargs; i1++) {
 467       const Type* t = domain_sig->field_at(TypeFunc::Parms + i1);
 468       if (method()->has_scalarized_args() && t->is_valuetypeptr() && !t->maybe_null()) {
 469         // Value type arguments are not passed by reference: we get an argument per
 470         // field of the value type. Build ValueTypeNodes from the value type arguments.
 471         GraphKit arg_kit(jvms, &gvn);
 472         arg_kit.set_control(map->control());
 473         ValueTypeNode* vt = ValueTypeNode::make_from_multi(&arg_kit, call, sig_cc, t->value_klass(), j, true);
 474         map->set_control(arg_kit.control());
 475         map->set_argument(jvms, i1, vt);
 476       } else {
 477         map->set_argument(jvms, i1, call->in(j++));
 478         BasicType bt = t->basic_type();
 479         while (SigEntry::next_is_reserved(sig_cc, bt, true)) {
 480           j += type2size[bt]; // Skip reserved arguments
 481         }
 482       }
 483     }
 484 
 485     C->print_inlining_assert_ready();
 486 
 487     C->print_inlining_move_to(this);
 488 
 489     C->log_late_inline(this);
 490 
 491     // This check is done here because for_method_handle_inline() method
 492     // needs jvms for inlined state.
 493     if (!do_late_inline_check(jvms)) {
 494       map->disconnect_inputs(NULL, C);
 495       return;
 496     }
 497 
 498     // Setup default node notes to be picked up by the inlining
 499     Node_Notes* old_nn = C->node_notes_at(call->_idx);
 500     if (old_nn != NULL) {
 501       Node_Notes* entry_nn = old_nn->clone(C);
 502       entry_nn->set_jvms(jvms);
 503       C->set_default_node_notes(entry_nn);
 504     }
 505 
 506     // Now perform the inlining using the synthesized JVMState
 507     JVMState* new_jvms = _inline_cg->generate(jvms);
 508     if (new_jvms == NULL)  return;  // no change
 509     if (C->failing())      return;
 510 
 511     // Capture any exceptional control flow
 512     GraphKit kit(new_jvms);
 513 
 514     // Find the result object
 515     Node* result = C->top();
 516     int   result_size = method()->return_type()->size();
 517     if (result_size != 0 && !kit.stopped()) {
 518       result = (result_size == 1) ? kit.pop() : kit.pop_pair();
 519     }
 520 
 521     C->set_has_loops(C->has_loops() || _inline_cg->method()->has_loops());
 522     C->env()->notice_inlined_method(_inline_cg->method());
 523     C->set_inlining_progress(true);
 524     C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup
 525 
 526     // Handle value type returns
 527     bool returned_as_fields = call->tf()->returns_value_type_as_fields();
 528     if (result->is_ValueType()) {
 529       ValueTypeNode* vt = result->as_ValueType();
 530       if (returned_as_fields) {
 531         // Return of multiple values (the fields of a value type)
 532         vt->replace_call_results(&kit, call, C);
 533         if (vt->is_allocated(&gvn) && !StressValueTypeReturnedAsFields) {
 534           result = vt->get_oop();
 535         } else {
 536           result = vt->tagged_klass(gvn);
 537         }
 538       } else {
 539         result = ValueTypePtrNode::make_from_value_type(&kit, vt);
 540       }
 541     } else if (gvn.type(result)->is_valuetypeptr() && returned_as_fields) {
 542       const Type* vt_t = call->_tf->range_sig()->field_at(TypeFunc::Parms);
 543       Node* cast = new CheckCastPPNode(NULL, result, vt_t);
 544       gvn.record_for_igvn(cast);
 545       ValueTypePtrNode* vtptr = ValueTypePtrNode::make_from_oop(&kit, gvn.transform(cast));
 546       vtptr->replace_call_results(&kit, call, C);
 547       result = cast;
 548     }
 549 
 550     kit.replace_call(call, result, true);
 551   }
 552 }
 553 
 554 
 555 CallGenerator* CallGenerator::for_late_inline(ciMethod* method, CallGenerator* inline_cg) {
 556   return new LateInlineCallGenerator(method, inline_cg);
 557 }
 558 
 559 class LateInlineMHCallGenerator : public LateInlineCallGenerator {
 560   ciMethod* _caller;
 561   int _attempt;
 562   bool _input_not_const;
 563 
 564   virtual bool do_late_inline_check(JVMState* jvms);
 565   virtual bool already_attempted() const { return _attempt > 0; }
 566 
 567  public:
 568   LateInlineMHCallGenerator(ciMethod* caller, ciMethod* callee, bool input_not_const) :
 569     LateInlineCallGenerator(callee, NULL), _caller(caller), _attempt(0), _input_not_const(input_not_const) {}
 570 
 571   virtual bool is_mh_late_inline() const { return true; }
 572 
 573   virtual JVMState* generate(JVMState* jvms) {
 574     JVMState* new_jvms = LateInlineCallGenerator::generate(jvms);
 575 
 576     Compile* C = Compile::current();
 577     if (_input_not_const) {
 578       // inlining won't be possible so no need to enqueue right now.
 579       call_node()->set_generator(this);
 580     } else {
 581       C->add_late_inline(this);
 582     }
 583     return new_jvms;
 584   }
 585 };
 586 
 587 bool LateInlineMHCallGenerator::do_late_inline_check(JVMState* jvms) {
 588 
 589   CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), _input_not_const, AlwaysIncrementalInline);
 590 
 591   Compile::current()->print_inlining_update_delayed(this);
 592 
 593   if (!_input_not_const) {
 594     _attempt++;
 595   }
 596 
 597   if (cg != NULL && (cg->is_inline() || cg->is_inlined_method_handle_intrinsic(jvms, cg->method()))) {
 598     assert(!cg->is_late_inline(), "we're doing late inlining");
 599     _inline_cg = cg;
 600     Compile::current()->dec_number_of_mh_late_inlines();
 601     return true;
 602   }
 603 
 604   call_node()->set_generator(this);
 605   return false;
 606 }
 607 
 608 CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const) {
 609   Compile::current()->inc_number_of_mh_late_inlines();
 610   CallGenerator* cg = new LateInlineMHCallGenerator(caller, callee, input_not_const);
 611   return cg;
 612 }
 613 
 614 class LateInlineStringCallGenerator : public LateInlineCallGenerator {
 615 
 616  public:
 617   LateInlineStringCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
 618     LateInlineCallGenerator(method, inline_cg) {}
 619 
 620   virtual JVMState* generate(JVMState* jvms) {
 621     Compile *C = Compile::current();
 622 
 623     C->log_inline_id(this);
 624 
 625     C->add_string_late_inline(this);
 626 
 627     JVMState* new_jvms =  DirectCallGenerator::generate(jvms);
 628     return new_jvms;
 629   }
 630 
 631   virtual bool is_string_late_inline() const { return true; }
 632 };
 633 
 634 CallGenerator* CallGenerator::for_string_late_inline(ciMethod* method, CallGenerator* inline_cg) {
 635   return new LateInlineStringCallGenerator(method, inline_cg);
 636 }
 637 
 638 class LateInlineBoxingCallGenerator : public LateInlineCallGenerator {
 639 
 640  public:
 641   LateInlineBoxingCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
 642     LateInlineCallGenerator(method, inline_cg, /*is_pure=*/true) {}
 643 
 644   virtual JVMState* generate(JVMState* jvms) {
 645     Compile *C = Compile::current();
 646 
 647     C->log_inline_id(this);
 648 
 649     C->add_boxing_late_inline(this);
 650 
 651     JVMState* new_jvms =  DirectCallGenerator::generate(jvms);
 652     return new_jvms;
 653   }
 654 };
 655 
 656 CallGenerator* CallGenerator::for_boxing_late_inline(ciMethod* method, CallGenerator* inline_cg) {
 657   return new LateInlineBoxingCallGenerator(method, inline_cg);
 658 }
 659 
 660 //---------------------------WarmCallGenerator--------------------------------
 661 // Internal class which handles initial deferral of inlining decisions.
 662 class WarmCallGenerator : public CallGenerator {
 663   WarmCallInfo*   _call_info;
 664   CallGenerator*  _if_cold;
 665   CallGenerator*  _if_hot;
 666   bool            _is_virtual;   // caches virtuality of if_cold
 667   bool            _is_inline;    // caches inline-ness of if_hot
 668 
 669 public:
 670   WarmCallGenerator(WarmCallInfo* ci,
 671                     CallGenerator* if_cold,
 672                     CallGenerator* if_hot)
 673     : CallGenerator(if_cold->method())
 674   {
 675     assert(method() == if_hot->method(), "consistent choices");
 676     _call_info  = ci;
 677     _if_cold    = if_cold;
 678     _if_hot     = if_hot;
 679     _is_virtual = if_cold->is_virtual();
 680     _is_inline  = if_hot->is_inline();
 681   }
 682 
 683   virtual bool      is_inline() const           { return _is_inline; }
 684   virtual bool      is_virtual() const          { return _is_virtual; }
 685   virtual bool      is_deferred() const         { return true; }
 686 
 687   virtual JVMState* generate(JVMState* jvms);
 688 };
 689 
 690 
 691 CallGenerator* CallGenerator::for_warm_call(WarmCallInfo* ci,
 692                                             CallGenerator* if_cold,
 693                                             CallGenerator* if_hot) {
 694   return new WarmCallGenerator(ci, if_cold, if_hot);
 695 }
 696 
 697 JVMState* WarmCallGenerator::generate(JVMState* jvms) {
 698   Compile* C = Compile::current();
 699   C->print_inlining_update(this);
 700 
 701   if (C->log() != NULL) {
 702     C->log()->elem("warm_call bci='%d'", jvms->bci());
 703   }
 704   jvms = _if_cold->generate(jvms);
 705   if (jvms != NULL) {
 706     Node* m = jvms->map()->control();
 707     if (m->is_CatchProj()) m = m->in(0);  else m = C->top();
 708     if (m->is_Catch())     m = m->in(0);  else m = C->top();
 709     if (m->is_Proj())      m = m->in(0);  else m = C->top();
 710     if (m->is_CallJava()) {
 711       _call_info->set_call(m->as_Call());
 712       _call_info->set_hot_cg(_if_hot);
 713 #ifndef PRODUCT
 714       if (PrintOpto || PrintOptoInlining) {
 715         tty->print_cr("Queueing for warm inlining at bci %d:", jvms->bci());
 716         tty->print("WCI: ");
 717         _call_info->print();
 718       }
 719 #endif
 720       _call_info->set_heat(_call_info->compute_heat());
 721       C->set_warm_calls(_call_info->insert_into(C->warm_calls()));
 722     }
 723   }
 724   return jvms;
 725 }
 726 
 727 void WarmCallInfo::make_hot() {
 728   Unimplemented();
 729 }
 730 
 731 void WarmCallInfo::make_cold() {
 732   // No action:  Just dequeue.
 733 }
 734 
 735 
 736 //------------------------PredictedCallGenerator------------------------------
 737 // Internal class which handles all out-of-line calls checking receiver type.
 738 class PredictedCallGenerator : public CallGenerator {
 739   ciKlass*       _predicted_receiver;
 740   CallGenerator* _if_missed;
 741   CallGenerator* _if_hit;
 742   float          _hit_prob;
 743   bool           _exact_check;
 744 
 745 public:
 746   PredictedCallGenerator(ciKlass* predicted_receiver,
 747                          CallGenerator* if_missed,
 748                          CallGenerator* if_hit, bool exact_check,
 749                          float hit_prob)
 750     : CallGenerator(if_missed->method())
 751   {
 752     // The call profile data may predict the hit_prob as extreme as 0 or 1.
 753     // Remove the extremes values from the range.
 754     if (hit_prob > PROB_MAX)   hit_prob = PROB_MAX;
 755     if (hit_prob < PROB_MIN)   hit_prob = PROB_MIN;
 756 
 757     _predicted_receiver = predicted_receiver;
 758     _if_missed          = if_missed;
 759     _if_hit             = if_hit;
 760     _hit_prob           = hit_prob;
 761     _exact_check        = exact_check;
 762   }
 763 
 764   virtual bool      is_virtual()   const    { return true; }
 765   virtual bool      is_inline()    const    { return _if_hit->is_inline(); }
 766   virtual bool      is_deferred()  const    { return _if_hit->is_deferred(); }
 767 
 768   virtual JVMState* generate(JVMState* jvms);
 769 };
 770 
 771 
 772 CallGenerator* CallGenerator::for_predicted_call(ciKlass* predicted_receiver,
 773                                                  CallGenerator* if_missed,
 774                                                  CallGenerator* if_hit,
 775                                                  float hit_prob) {
 776   return new PredictedCallGenerator(predicted_receiver, if_missed, if_hit,
 777                                     /*exact_check=*/true, hit_prob);
 778 }
 779 
 780 CallGenerator* CallGenerator::for_guarded_call(ciKlass* guarded_receiver,
 781                                                CallGenerator* if_missed,
 782                                                CallGenerator* if_hit) {
 783   return new PredictedCallGenerator(guarded_receiver, if_missed, if_hit,
 784                                     /*exact_check=*/false, PROB_ALWAYS);
 785 }
 786 
 787 JVMState* PredictedCallGenerator::generate(JVMState* jvms) {
 788   GraphKit kit(jvms);
 789   kit.C->print_inlining_update(this);
 790   PhaseGVN& gvn = kit.gvn();
 791   // We need an explicit receiver null_check before checking its type.
 792   // We share a map with the caller, so his JVMS gets adjusted.
 793   Node* receiver = kit.argument(0);
 794   CompileLog* log = kit.C->log();
 795   if (log != NULL) {
 796     log->elem("predicted_call bci='%d' exact='%d' klass='%d'",
 797               jvms->bci(), (_exact_check ? 1 : 0), log->identify(_predicted_receiver));
 798   }
 799 
 800   receiver = kit.null_check_receiver_before_call(method());
 801   if (kit.stopped()) {
 802     return kit.transfer_exceptions_into_jvms();
 803   }
 804 
 805   // Make a copy of the replaced nodes in case we need to restore them
 806   ReplacedNodes replaced_nodes = kit.map()->replaced_nodes();
 807   replaced_nodes.clone();
 808 
 809   Node* casted_receiver = receiver;  // will get updated in place...
 810   Node* slow_ctl = NULL;
 811   if (_exact_check) {
 812     slow_ctl = kit.type_check_receiver(receiver, _predicted_receiver, _hit_prob,
 813                                        &casted_receiver);
 814   } else {
 815     slow_ctl = kit.subtype_check_receiver(receiver, _predicted_receiver,
 816                                           &casted_receiver);
 817   }
 818 
 819   SafePointNode* slow_map = NULL;
 820   JVMState* slow_jvms = NULL;
 821   { PreserveJVMState pjvms(&kit);
 822     kit.set_control(slow_ctl);
 823     if (!kit.stopped()) {
 824       slow_jvms = _if_missed->generate(kit.sync_jvms());
 825       if (kit.failing())
 826         return NULL;  // might happen because of NodeCountInliningCutoff
 827       assert(slow_jvms != NULL, "must be");
 828       kit.add_exception_states_from(slow_jvms);
 829       kit.set_map(slow_jvms->map());
 830       if (!kit.stopped())
 831         slow_map = kit.stop();
 832     }
 833   }
 834 
 835   if (kit.stopped()) {
 836     // Instance exactly does not matches the desired type.
 837     kit.set_jvms(slow_jvms);
 838     return kit.transfer_exceptions_into_jvms();
 839   }
 840 
 841   // fall through if the instance exactly matches the desired type
 842   kit.replace_in_map(receiver, casted_receiver);
 843 
 844   // Make the hot call:
 845   JVMState* new_jvms = _if_hit->generate(kit.sync_jvms());
 846   if (new_jvms == NULL) {
 847     // Inline failed, so make a direct call.
 848     assert(_if_hit->is_inline(), "must have been a failed inline");
 849     CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method());
 850     new_jvms = cg->generate(kit.sync_jvms());
 851   }
 852   kit.add_exception_states_from(new_jvms);
 853   kit.set_jvms(new_jvms);
 854 
 855   // Need to merge slow and fast?
 856   if (slow_map == NULL) {
 857     // The fast path is the only path remaining.
 858     return kit.transfer_exceptions_into_jvms();
 859   }
 860 
 861   if (kit.stopped()) {
 862     // Inlined method threw an exception, so it's just the slow path after all.
 863     kit.set_jvms(slow_jvms);
 864     return kit.transfer_exceptions_into_jvms();
 865   }
 866 
 867   // Allocate value types if they are merged with objects (similar to Parse::merge_common())
 868   uint tos = kit.jvms()->stkoff() + kit.sp();
 869   uint limit = slow_map->req();
 870   for (uint i = TypeFunc::Parms; i < limit; i++) {
 871     Node* m = kit.map()->in(i);
 872     Node* n = slow_map->in(i);
 873     const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
 874     if (m->is_ValueType() && !t->isa_valuetype()) {
 875       // Allocate value type in fast path
 876       m = ValueTypePtrNode::make_from_value_type(&kit, m->as_ValueType());
 877       kit.map()->set_req(i, m);
 878     }
 879     if (n->is_ValueType() && !t->isa_valuetype()) {
 880       // Allocate value type in slow path
 881       PreserveJVMState pjvms(&kit);
 882       kit.set_map(slow_map);
 883       n = ValueTypePtrNode::make_from_value_type(&kit, n->as_ValueType());
 884       kit.map()->set_req(i, n);
 885       slow_map = kit.stop();
 886     }
 887   }
 888 
 889   // There are 2 branches and the replaced nodes are only valid on
 890   // one: restore the replaced nodes to what they were before the
 891   // branch.
 892   kit.map()->set_replaced_nodes(replaced_nodes);
 893 
 894   // Finish the diamond.
 895   kit.C->set_has_split_ifs(true); // Has chance for split-if optimization
 896   RegionNode* region = new RegionNode(3);
 897   region->init_req(1, kit.control());
 898   region->init_req(2, slow_map->control());
 899   kit.set_control(gvn.transform(region));
 900   Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO);
 901   iophi->set_req(2, slow_map->i_o());
 902   kit.set_i_o(gvn.transform(iophi));
 903   // Merge memory
 904   kit.merge_memory(slow_map->merged_memory(), region, 2);
 905   // Transform new memory Phis.
 906   for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) {
 907     Node* phi = mms.memory();
 908     if (phi->is_Phi() && phi->in(0) == region) {
 909       mms.set_memory(gvn.transform(phi));
 910     }
 911   }
 912   for (uint i = TypeFunc::Parms; i < limit; i++) {
 913     // Skip unused stack slots; fast forward to monoff();
 914     if (i == tos) {
 915       i = kit.jvms()->monoff();
 916       if( i >= limit ) break;
 917     }
 918     Node* m = kit.map()->in(i);
 919     Node* n = slow_map->in(i);
 920     if (m != n) {
 921       const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
 922       Node* phi = PhiNode::make(region, m, t);
 923       phi->set_req(2, n);
 924       kit.map()->set_req(i, gvn.transform(phi));
 925     }
 926   }
 927   return kit.transfer_exceptions_into_jvms();
 928 }
 929 
 930 
 931 CallGenerator* CallGenerator::for_method_handle_call(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool delayed_forbidden) {
 932   assert(callee->is_method_handle_intrinsic(), "for_method_handle_call mismatch");
 933   bool input_not_const;
 934   CallGenerator* cg = CallGenerator::for_method_handle_inline(jvms, caller, callee, input_not_const, false);
 935   Compile* C = Compile::current();
 936   if (cg != NULL) {
 937     if (!delayed_forbidden && AlwaysIncrementalInline) {
 938       return CallGenerator::for_late_inline(callee, cg);
 939     } else {
 940       return cg;
 941     }
 942   }
 943   int bci = jvms->bci();
 944   ciCallProfile profile = caller->call_profile_at_bci(bci);
 945   int call_site_count = caller->scale_count(profile.count());
 946 
 947   if (IncrementalInline && (AlwaysIncrementalInline ||
 948                             (call_site_count > 0 && (input_not_const || !C->inlining_incrementally() || C->over_inlining_cutoff())))) {
 949     return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
 950   } else {
 951     // Out-of-line call.
 952     return CallGenerator::for_direct_call(callee);
 953   }
 954 }
 955 
 956 static void cast_argument(int nargs, int arg_nb, ciType* t, GraphKit& kit) {
 957   PhaseGVN& gvn = kit.gvn();
 958   Node* arg = kit.argument(arg_nb);
 959   const Type* arg_type = arg->bottom_type();
 960   const Type* sig_type = TypeOopPtr::make_from_klass(t->as_klass());
 961   if (arg_type->isa_oopptr() && !arg_type->higher_equal(sig_type)) {
 962     const Type* narrowed_arg_type = arg_type->join_speculative(sig_type); // keep speculative part
 963     arg = gvn.transform(new CheckCastPPNode(kit.control(), arg, narrowed_arg_type));
 964     kit.set_argument(arg_nb, arg);
 965   }
 966   if (sig_type->is_valuetypeptr() && !arg->is_ValueType() &&
 967       !kit.gvn().type(arg)->maybe_null() && t->as_value_klass()->is_scalarizable()) {
 968     arg = ValueTypeNode::make_from_oop(&kit, arg, t->as_value_klass());
 969     kit.set_argument(arg_nb, arg);
 970   }
 971 }
 972 
 973 CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool& input_not_const, bool delayed_forbidden) {
 974   GraphKit kit(jvms);
 975   PhaseGVN& gvn = kit.gvn();
 976   Compile* C = kit.C;
 977   vmIntrinsics::ID iid = callee->intrinsic_id();
 978   input_not_const = true;
 979   switch (iid) {
 980   case vmIntrinsics::_invokeBasic:
 981     {
 982       // Get MethodHandle receiver:
 983       Node* receiver = kit.argument(0);
 984       if (receiver->Opcode() == Op_ConP) {
 985         input_not_const = false;
 986         const TypeOopPtr* oop_ptr = receiver->bottom_type()->is_oopptr();
 987         ciMethod* target = oop_ptr->const_oop()->as_method_handle()->get_vmtarget();
 988         const int vtable_index = Method::invalid_vtable_index;
 989 
 990         if (!ciMethod::is_consistent_info(callee, target)) {
 991           print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
 992                                  "signatures mismatch");
 993           return NULL;
 994         }
 995 
 996         CallGenerator* cg = C->call_generator(target, vtable_index,
 997                                               false /* call_does_dispatch */,
 998                                               jvms,
 999                                               true /* allow_inline */,
1000                                               PROB_ALWAYS,
1001                                               NULL,
1002                                               true,
1003                                               delayed_forbidden);
1004         return cg;
1005       } else {
1006         print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1007                                "receiver not constant");
1008       }
1009     }
1010     break;
1011 
1012   case vmIntrinsics::_linkToVirtual:
1013   case vmIntrinsics::_linkToStatic:
1014   case vmIntrinsics::_linkToSpecial:
1015   case vmIntrinsics::_linkToInterface:
1016     {
1017       int nargs = callee->arg_size();
1018       // Get MemberName argument:
1019       Node* member_name = kit.argument(nargs - 1);
1020       if (member_name->Opcode() == Op_ConP) {
1021         input_not_const = false;
1022         const TypeOopPtr* oop_ptr = member_name->bottom_type()->is_oopptr();
1023         ciMethod* target = oop_ptr->const_oop()->as_member_name()->get_vmtarget();
1024 
1025         if (!ciMethod::is_consistent_info(callee, target)) {
1026           print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1027                                  "signatures mismatch");
1028           return NULL;
1029         }
1030 
1031         // In lambda forms we erase signature types to avoid resolving issues
1032         // involving class loaders.  When we optimize a method handle invoke
1033         // to a direct call we must cast the receiver and arguments to its
1034         // actual types.
1035         ciSignature* signature = target->signature();
1036         const int receiver_skip = target->is_static() ? 0 : 1;
1037         // Cast receiver to its type.
1038         if (!target->is_static()) {
1039           cast_argument(nargs, 0, signature->accessing_klass(), kit);
1040         }
1041         // Cast reference arguments to its type.
1042         for (int i = 0, j = 0; i < signature->count(); i++) {
1043           ciType* t = signature->type_at(i);
1044           if (t->is_klass()) {
1045             cast_argument(nargs, receiver_skip + j, t, kit);
1046           }
1047           j += t->size();  // long and double take two slots
1048         }
1049 
1050         // Try to get the most accurate receiver type
1051         const bool is_virtual              = (iid == vmIntrinsics::_linkToVirtual);
1052         const bool is_virtual_or_interface = (is_virtual || iid == vmIntrinsics::_linkToInterface);
1053         int  vtable_index       = Method::invalid_vtable_index;
1054         bool call_does_dispatch = false;
1055 
1056         ciKlass* speculative_receiver_type = NULL;
1057         if (is_virtual_or_interface) {
1058           ciInstanceKlass* klass = target->holder();
1059           Node*             receiver_node = kit.argument(0);
1060           const TypeOopPtr* receiver_type = gvn.type(receiver_node)->isa_oopptr();
1061           // call_does_dispatch and vtable_index are out-parameters.  They might be changed.
1062           // optimize_virtual_call() takes 2 different holder
1063           // arguments for a corner case that doesn't apply here (see
1064           // Parse::do_call())
1065           target = C->optimize_virtual_call(caller, jvms->bci(), klass, klass,
1066                                             target, receiver_type, is_virtual,
1067                                             call_does_dispatch, vtable_index, // out-parameters
1068                                             false /* check_access */);
1069           // We lack profiling at this call but type speculation may
1070           // provide us with a type
1071           speculative_receiver_type = (receiver_type != NULL) ? receiver_type->speculative_type() : NULL;
1072         }
1073         CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms,
1074                                               !StressMethodHandleLinkerInlining /* allow_inline */,
1075                                               PROB_ALWAYS,
1076                                               speculative_receiver_type,
1077                                               true,
1078                                               delayed_forbidden);
1079         return cg;
1080       } else {
1081         print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1082                                "member_name not constant");
1083       }
1084     }
1085     break;
1086 
1087   default:
1088     fatal("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid));
1089     break;
1090   }
1091   return NULL;
1092 }
1093 
1094 
1095 //------------------------PredicatedIntrinsicGenerator------------------------------
1096 // Internal class which handles all predicated Intrinsic calls.
1097 class PredicatedIntrinsicGenerator : public CallGenerator {
1098   CallGenerator* _intrinsic;
1099   CallGenerator* _cg;
1100 
1101 public:
1102   PredicatedIntrinsicGenerator(CallGenerator* intrinsic,
1103                                CallGenerator* cg)
1104     : CallGenerator(cg->method())
1105   {
1106     _intrinsic = intrinsic;
1107     _cg        = cg;
1108   }
1109 
1110   virtual bool      is_virtual()   const    { return true; }
1111   virtual bool      is_inlined()   const    { return true; }
1112   virtual bool      is_intrinsic() const    { return true; }
1113 
1114   virtual JVMState* generate(JVMState* jvms);
1115 };
1116 
1117 
1118 CallGenerator* CallGenerator::for_predicated_intrinsic(CallGenerator* intrinsic,
1119                                                        CallGenerator* cg) {
1120   return new PredicatedIntrinsicGenerator(intrinsic, cg);
1121 }
1122 
1123 
1124 JVMState* PredicatedIntrinsicGenerator::generate(JVMState* jvms) {
1125   // The code we want to generate here is:
1126   //    if (receiver == NULL)
1127   //        uncommon_Trap
1128   //    if (predicate(0))
1129   //        do_intrinsic(0)
1130   //    else
1131   //    if (predicate(1))
1132   //        do_intrinsic(1)
1133   //    ...
1134   //    else
1135   //        do_java_comp
1136 
1137   GraphKit kit(jvms);
1138   PhaseGVN& gvn = kit.gvn();
1139 
1140   CompileLog* log = kit.C->log();
1141   if (log != NULL) {
1142     log->elem("predicated_intrinsic bci='%d' method='%d'",
1143               jvms->bci(), log->identify(method()));
1144   }
1145 
1146   if (!method()->is_static()) {
1147     // We need an explicit receiver null_check before checking its type in predicate.
1148     // We share a map with the caller, so his JVMS gets adjusted.
1149     kit.null_check_receiver_before_call(method());
1150     if (kit.stopped()) {
1151       return kit.transfer_exceptions_into_jvms();
1152     }
1153   }
1154 
1155   int n_predicates = _intrinsic->predicates_count();
1156   assert(n_predicates > 0, "sanity");
1157 
1158   JVMState** result_jvms = NEW_RESOURCE_ARRAY(JVMState*, (n_predicates+1));
1159 
1160   // Region for normal compilation code if intrinsic failed.
1161   Node* slow_region = new RegionNode(1);
1162 
1163   int results = 0;
1164   for (int predicate = 0; (predicate < n_predicates) && !kit.stopped(); predicate++) {
1165 #ifdef ASSERT
1166     JVMState* old_jvms = kit.jvms();
1167     SafePointNode* old_map = kit.map();
1168     Node* old_io  = old_map->i_o();
1169     Node* old_mem = old_map->memory();
1170     Node* old_exc = old_map->next_exception();
1171 #endif
1172     Node* else_ctrl = _intrinsic->generate_predicate(kit.sync_jvms(), predicate);
1173 #ifdef ASSERT
1174     // Assert(no_new_memory && no_new_io && no_new_exceptions) after generate_predicate.
1175     assert(old_jvms == kit.jvms(), "generate_predicate should not change jvm state");
1176     SafePointNode* new_map = kit.map();
1177     assert(old_io  == new_map->i_o(), "generate_predicate should not change i_o");
1178     assert(old_mem == new_map->memory(), "generate_predicate should not change memory");
1179     assert(old_exc == new_map->next_exception(), "generate_predicate should not add exceptions");
1180 #endif
1181     if (!kit.stopped()) {
1182       PreserveJVMState pjvms(&kit);
1183       // Generate intrinsic code:
1184       JVMState* new_jvms = _intrinsic->generate(kit.sync_jvms());
1185       if (new_jvms == NULL) {
1186         // Intrinsic failed, use normal compilation path for this predicate.
1187         slow_region->add_req(kit.control());
1188       } else {
1189         kit.add_exception_states_from(new_jvms);
1190         kit.set_jvms(new_jvms);
1191         if (!kit.stopped()) {
1192           result_jvms[results++] = kit.jvms();
1193         }
1194       }
1195     }
1196     if (else_ctrl == NULL) {
1197       else_ctrl = kit.C->top();
1198     }
1199     kit.set_control(else_ctrl);
1200   }
1201   if (!kit.stopped()) {
1202     // Final 'else' after predicates.
1203     slow_region->add_req(kit.control());
1204   }
1205   if (slow_region->req() > 1) {
1206     PreserveJVMState pjvms(&kit);
1207     // Generate normal compilation code:
1208     kit.set_control(gvn.transform(slow_region));
1209     JVMState* new_jvms = _cg->generate(kit.sync_jvms());
1210     if (kit.failing())
1211       return NULL;  // might happen because of NodeCountInliningCutoff
1212     assert(new_jvms != NULL, "must be");
1213     kit.add_exception_states_from(new_jvms);
1214     kit.set_jvms(new_jvms);
1215     if (!kit.stopped()) {
1216       result_jvms[results++] = kit.jvms();
1217     }
1218   }
1219 
1220   if (results == 0) {
1221     // All paths ended in uncommon traps.
1222     (void) kit.stop();
1223     return kit.transfer_exceptions_into_jvms();
1224   }
1225 
1226   if (results == 1) { // Only one path
1227     kit.set_jvms(result_jvms[0]);
1228     return kit.transfer_exceptions_into_jvms();
1229   }
1230 
1231   // Merge all paths.
1232   kit.C->set_has_split_ifs(true); // Has chance for split-if optimization
1233   RegionNode* region = new RegionNode(results + 1);
1234   Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO);
1235   for (int i = 0; i < results; i++) {
1236     JVMState* jvms = result_jvms[i];
1237     int path = i + 1;
1238     SafePointNode* map = jvms->map();
1239     region->init_req(path, map->control());
1240     iophi->set_req(path, map->i_o());
1241     if (i == 0) {
1242       kit.set_jvms(jvms);
1243     } else {
1244       kit.merge_memory(map->merged_memory(), region, path);
1245     }
1246   }
1247   kit.set_control(gvn.transform(region));
1248   kit.set_i_o(gvn.transform(iophi));
1249   // Transform new memory Phis.
1250   for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) {
1251     Node* phi = mms.memory();
1252     if (phi->is_Phi() && phi->in(0) == region) {
1253       mms.set_memory(gvn.transform(phi));
1254     }
1255   }
1256 
1257   // Merge debug info.
1258   Node** ins = NEW_RESOURCE_ARRAY(Node*, results);
1259   uint tos = kit.jvms()->stkoff() + kit.sp();
1260   Node* map = kit.map();
1261   uint limit = map->req();
1262   for (uint i = TypeFunc::Parms; i < limit; i++) {
1263     // Skip unused stack slots; fast forward to monoff();
1264     if (i == tos) {
1265       i = kit.jvms()->monoff();
1266       if( i >= limit ) break;
1267     }
1268     Node* n = map->in(i);
1269     ins[0] = n;
1270     const Type* t = gvn.type(n);
1271     bool needs_phi = false;
1272     for (int j = 1; j < results; j++) {
1273       JVMState* jvms = result_jvms[j];
1274       Node* jmap = jvms->map();
1275       Node* m = NULL;
1276       if (jmap->req() > i) {
1277         m = jmap->in(i);
1278         if (m != n) {
1279           needs_phi = true;
1280           t = t->meet_speculative(gvn.type(m));
1281         }
1282       }
1283       ins[j] = m;
1284     }
1285     if (needs_phi) {
1286       Node* phi = PhiNode::make(region, n, t);
1287       for (int j = 1; j < results; j++) {
1288         phi->set_req(j + 1, ins[j]);
1289       }
1290       map->set_req(i, gvn.transform(phi));
1291     }
1292   }
1293 
1294   return kit.transfer_exceptions_into_jvms();
1295 }
1296 
1297 //-------------------------UncommonTrapCallGenerator-----------------------------
1298 // Internal class which handles all out-of-line calls checking receiver type.
1299 class UncommonTrapCallGenerator : public CallGenerator {
1300   Deoptimization::DeoptReason _reason;
1301   Deoptimization::DeoptAction _action;
1302 
1303 public:
1304   UncommonTrapCallGenerator(ciMethod* m,
1305                             Deoptimization::DeoptReason reason,
1306                             Deoptimization::DeoptAction action)
1307     : CallGenerator(m)
1308   {
1309     _reason = reason;
1310     _action = action;
1311   }
1312 
1313   virtual bool      is_virtual() const          { ShouldNotReachHere(); return false; }
1314   virtual bool      is_trap() const             { return true; }
1315 
1316   virtual JVMState* generate(JVMState* jvms);
1317 };
1318 
1319 
1320 CallGenerator*
1321 CallGenerator::for_uncommon_trap(ciMethod* m,
1322                                  Deoptimization::DeoptReason reason,
1323                                  Deoptimization::DeoptAction action) {
1324   return new UncommonTrapCallGenerator(m, reason, action);
1325 }
1326 
1327 
1328 JVMState* UncommonTrapCallGenerator::generate(JVMState* jvms) {
1329   GraphKit kit(jvms);
1330   kit.C->print_inlining_update(this);
1331   // Take the trap with arguments pushed on the stack.  (Cf. null_check_receiver).
1332   // Callsite signature can be different from actual method being called (i.e _linkTo* sites).
1333   // Use callsite signature always.
1334   ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci());
1335   int nargs = declared_method->arg_size();
1336   kit.inc_sp(nargs);
1337   assert(nargs <= kit.sp() && kit.sp() <= jvms->stk_size(), "sane sp w/ args pushed");
1338   if (_reason == Deoptimization::Reason_class_check &&
1339       _action == Deoptimization::Action_maybe_recompile) {
1340     // Temp fix for 6529811
1341     // Don't allow uncommon_trap to override our decision to recompile in the event
1342     // of a class cast failure for a monomorphic call as it will never let us convert
1343     // the call to either bi-morphic or megamorphic and can lead to unc-trap loops
1344     bool keep_exact_action = true;
1345     kit.uncommon_trap(_reason, _action, NULL, "monomorphic vcall checkcast", false, keep_exact_action);
1346   } else {
1347     kit.uncommon_trap(_reason, _action);
1348   }
1349   return kit.transfer_exceptions_into_jvms();
1350 }
1351 
1352 // (Note:  Moved hook_up_call to GraphKit::set_edges_for_java_call.)
1353 
1354 // (Node:  Merged hook_up_exits into ParseGenerator::generate.)
1355 
1356 #define NODES_OVERHEAD_PER_METHOD (30.0)
1357 #define NODES_PER_BYTECODE (9.5)
1358 
1359 void WarmCallInfo::init(JVMState* call_site, ciMethod* call_method, ciCallProfile& profile, float prof_factor) {
1360   int call_count = profile.count();
1361   int code_size = call_method->code_size();
1362 
1363   // Expected execution count is based on the historical count:
1364   _count = call_count < 0 ? 1 : call_site->method()->scale_count(call_count, prof_factor);
1365 
1366   // Expected profit from inlining, in units of simple call-overheads.
1367   _profit = 1.0;
1368 
1369   // Expected work performed by the call in units of call-overheads.
1370   // %%% need an empirical curve fit for "work" (time in call)
1371   float bytecodes_per_call = 3;
1372   _work = 1.0 + code_size / bytecodes_per_call;
1373 
1374   // Expected size of compilation graph:
1375   // -XX:+PrintParseStatistics once reported:
1376   //  Methods seen: 9184  Methods parsed: 9184  Nodes created: 1582391
1377   //  Histogram of 144298 parsed bytecodes:
1378   // %%% Need an better predictor for graph size.
1379   _size = NODES_OVERHEAD_PER_METHOD + (NODES_PER_BYTECODE * code_size);
1380 }
1381 
1382 // is_cold:  Return true if the node should never be inlined.
1383 // This is true if any of the key metrics are extreme.
1384 bool WarmCallInfo::is_cold() const {
1385   if (count()  <  WarmCallMinCount)        return true;
1386   if (profit() <  WarmCallMinProfit)       return true;
1387   if (work()   >  WarmCallMaxWork)         return true;
1388   if (size()   >  WarmCallMaxSize)         return true;
1389   return false;
1390 }
1391 
1392 // is_hot:  Return true if the node should be inlined immediately.
1393 // This is true if any of the key metrics are extreme.
1394 bool WarmCallInfo::is_hot() const {
1395   assert(!is_cold(), "eliminate is_cold cases before testing is_hot");
1396   if (count()  >= HotCallCountThreshold)   return true;
1397   if (profit() >= HotCallProfitThreshold)  return true;
1398   if (work()   <= HotCallTrivialWork)      return true;
1399   if (size()   <= HotCallTrivialSize)      return true;
1400   return false;
1401 }
1402 
1403 // compute_heat:
1404 float WarmCallInfo::compute_heat() const {
1405   assert(!is_cold(), "compute heat only on warm nodes");
1406   assert(!is_hot(),  "compute heat only on warm nodes");
1407   int min_size = MAX2(0,   (int)HotCallTrivialSize);
1408   int max_size = MIN2(500, (int)WarmCallMaxSize);
1409   float method_size = (size() - min_size) / MAX2(1, max_size - min_size);
1410   float size_factor;
1411   if      (method_size < 0.05)  size_factor = 4;   // 2 sigmas better than avg.
1412   else if (method_size < 0.15)  size_factor = 2;   // 1 sigma better than avg.
1413   else if (method_size < 0.5)   size_factor = 1;   // better than avg.
1414   else                          size_factor = 0.5; // worse than avg.
1415   return (count() * profit() * size_factor);
1416 }
1417 
1418 bool WarmCallInfo::warmer_than(WarmCallInfo* that) {
1419   assert(this != that, "compare only different WCIs");
1420   assert(this->heat() != 0 && that->heat() != 0, "call compute_heat 1st");
1421   if (this->heat() > that->heat())   return true;
1422   if (this->heat() < that->heat())   return false;
1423   assert(this->heat() == that->heat(), "no NaN heat allowed");
1424   // Equal heat.  Break the tie some other way.
1425   if (!this->call() || !that->call())  return (address)this > (address)that;
1426   return this->call()->_idx > that->call()->_idx;
1427 }
1428 
1429 //#define UNINIT_NEXT ((WarmCallInfo*)badAddress)
1430 #define UNINIT_NEXT ((WarmCallInfo*)NULL)
1431 
1432 WarmCallInfo* WarmCallInfo::insert_into(WarmCallInfo* head) {
1433   assert(next() == UNINIT_NEXT, "not yet on any list");
1434   WarmCallInfo* prev_p = NULL;
1435   WarmCallInfo* next_p = head;
1436   while (next_p != NULL && next_p->warmer_than(this)) {
1437     prev_p = next_p;
1438     next_p = prev_p->next();
1439   }
1440   // Install this between prev_p and next_p.
1441   this->set_next(next_p);
1442   if (prev_p == NULL)
1443     head = this;
1444   else
1445     prev_p->set_next(this);
1446   return head;
1447 }
1448 
1449 WarmCallInfo* WarmCallInfo::remove_from(WarmCallInfo* head) {
1450   WarmCallInfo* prev_p = NULL;
1451   WarmCallInfo* next_p = head;
1452   while (next_p != this) {
1453     assert(next_p != NULL, "this must be in the list somewhere");
1454     prev_p = next_p;
1455     next_p = prev_p->next();
1456   }
1457   next_p = this->next();
1458   debug_only(this->set_next(UNINIT_NEXT));
1459   // Remove this from between prev_p and next_p.
1460   if (prev_p == NULL)
1461     head = next_p;
1462   else
1463     prev_p->set_next(next_p);
1464   return head;
1465 }
1466 
1467 WarmCallInfo WarmCallInfo::_always_hot(WarmCallInfo::MAX_VALUE(), WarmCallInfo::MAX_VALUE(),
1468                                        WarmCallInfo::MIN_VALUE(), WarmCallInfo::MIN_VALUE());
1469 WarmCallInfo WarmCallInfo::_always_cold(WarmCallInfo::MIN_VALUE(), WarmCallInfo::MIN_VALUE(),
1470                                         WarmCallInfo::MAX_VALUE(), WarmCallInfo::MAX_VALUE());
1471 
1472 WarmCallInfo* WarmCallInfo::always_hot() {
1473   assert(_always_hot.is_hot(), "must always be hot");
1474   return &_always_hot;
1475 }
1476 
1477 WarmCallInfo* WarmCallInfo::always_cold() {
1478   assert(_always_cold.is_cold(), "must always be cold");
1479   return &_always_cold;
1480 }
1481 
1482 
1483 #ifndef PRODUCT
1484 
1485 void WarmCallInfo::print() const {
1486   tty->print("%s : C=%6.1f P=%6.1f W=%6.1f S=%6.1f H=%6.1f -> %p",
1487              is_cold() ? "cold" : is_hot() ? "hot " : "warm",
1488              count(), profit(), work(), size(), compute_heat(), next());
1489   tty->cr();
1490   if (call() != NULL)  call()->dump();
1491 }
1492 
1493 void print_wci(WarmCallInfo* ci) {
1494   ci->print();
1495 }
1496 
1497 void WarmCallInfo::print_all() const {
1498   for (const WarmCallInfo* p = this; p != NULL; p = p->next())
1499     p->print();
1500 }
1501 
1502 int WarmCallInfo::count_all() const {
1503   int cnt = 0;
1504   for (const WarmCallInfo* p = this; p != NULL; p = p->next())
1505     cnt++;
1506   return cnt;
1507 }
1508 
1509 #endif //PRODUCT