1 /* 2 * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "ci/bcEscapeAnalyzer.hpp" 27 #include "ci/ciCallSite.hpp" 28 #include "ci/ciObjArray.hpp" 29 #include "ci/ciMemberName.hpp" 30 #include "ci/ciMethodHandle.hpp" 31 #include "classfile/javaClasses.hpp" 32 #include "compiler/compileLog.hpp" 33 #include "opto/addnode.hpp" 34 #include "opto/callGenerator.hpp" 35 #include "opto/callnode.hpp" 36 #include "opto/castnode.hpp" 37 #include "opto/cfgnode.hpp" 38 #include "opto/inlinetypenode.hpp" 39 #include "opto/parse.hpp" 40 #include "opto/rootnode.hpp" 41 #include "opto/runtime.hpp" 42 #include "opto/subnode.hpp" 43 #include "runtime/os.inline.hpp" 44 #include "runtime/sharedRuntime.hpp" 45 #include "utilities/debug.hpp" 46 47 // Utility function. 48 const TypeFunc* CallGenerator::tf() const { 49 return TypeFunc::make(method()); 50 } 51 52 bool CallGenerator::is_inlined_method_handle_intrinsic(JVMState* jvms, ciMethod* m) { 53 return is_inlined_method_handle_intrinsic(jvms->method(), jvms->bci(), m); 54 } 55 56 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* caller, int bci, ciMethod* m) { 57 ciMethod* symbolic_info = caller->get_method_at_bci(bci); 58 return is_inlined_method_handle_intrinsic(symbolic_info, m); 59 } 60 61 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* symbolic_info, ciMethod* m) { 62 return symbolic_info->is_method_handle_intrinsic() && !m->is_method_handle_intrinsic(); 63 } 64 65 //-----------------------------ParseGenerator--------------------------------- 66 // Internal class which handles all direct bytecode traversal. 67 class ParseGenerator : public InlineCallGenerator { 68 private: 69 bool _is_osr; 70 float _expected_uses; 71 72 public: 73 ParseGenerator(ciMethod* method, float expected_uses, bool is_osr = false) 74 : InlineCallGenerator(method) 75 { 76 _is_osr = is_osr; 77 _expected_uses = expected_uses; 78 assert(InlineTree::check_can_parse(method) == nullptr, "parse must be possible"); 79 } 80 81 virtual bool is_parse() const { return true; } 82 virtual JVMState* generate(JVMState* jvms); 83 int is_osr() { return _is_osr; } 84 85 }; 86 87 JVMState* ParseGenerator::generate(JVMState* jvms) { 88 Compile* C = Compile::current(); 89 C->print_inlining_update(this); 90 91 if (is_osr()) { 92 // The JVMS for a OSR has a single argument (see its TypeFunc). 93 assert(jvms->depth() == 1, "no inline OSR"); 94 } 95 96 if (C->failing()) { 97 return nullptr; // bailing out of the compile; do not try to parse 98 } 99 100 Parse parser(jvms, method(), _expected_uses); 101 if (C->failing()) return nullptr; 102 103 // Grab signature for matching/allocation 104 GraphKit& exits = parser.exits(); 105 106 if (C->failing()) { 107 while (exits.pop_exception_state() != nullptr) ; 108 return nullptr; 109 } 110 111 assert(exits.jvms()->same_calls_as(jvms), "sanity"); 112 113 // Simply return the exit state of the parser, 114 // augmented by any exceptional states. 115 return exits.transfer_exceptions_into_jvms(); 116 } 117 118 //---------------------------DirectCallGenerator------------------------------ 119 // Internal class which handles all out-of-line calls w/o receiver type checks. 120 class DirectCallGenerator : public CallGenerator { 121 private: 122 CallStaticJavaNode* _call_node; 123 // Force separate memory and I/O projections for the exceptional 124 // paths to facilitate late inlining. 125 bool _separate_io_proj; 126 127 protected: 128 void set_call_node(CallStaticJavaNode* call) { _call_node = call; } 129 130 public: 131 DirectCallGenerator(ciMethod* method, bool separate_io_proj) 132 : CallGenerator(method), 133 _call_node(nullptr), 134 _separate_io_proj(separate_io_proj) 135 { 136 if (InlineTypeReturnedAsFields && method->is_method_handle_intrinsic()) { 137 // If that call has not been optimized by the time optimizations are over, 138 // we'll need to add a call to create an inline type instance from the klass 139 // returned by the call (see PhaseMacroExpand::expand_mh_intrinsic_return). 140 // Separating memory and I/O projections for exceptions is required to 141 // perform that graph transformation. 142 _separate_io_proj = true; 143 } 144 } 145 virtual JVMState* generate(JVMState* jvms); 146 147 virtual CallNode* call_node() const { return _call_node; } 148 virtual CallGenerator* with_call_node(CallNode* call) { 149 DirectCallGenerator* dcg = new DirectCallGenerator(method(), _separate_io_proj); 150 dcg->set_call_node(call->as_CallStaticJava()); 151 return dcg; 152 } 153 }; 154 155 JVMState* DirectCallGenerator::generate(JVMState* jvms) { 156 GraphKit kit(jvms); 157 kit.C->print_inlining_update(this); 158 PhaseGVN& gvn = kit.gvn(); 159 bool is_static = method()->is_static(); 160 address target = is_static ? SharedRuntime::get_resolve_static_call_stub() 161 : SharedRuntime::get_resolve_opt_virtual_call_stub(); 162 163 if (kit.C->log() != nullptr) { 164 kit.C->log()->elem("direct_call bci='%d'", jvms->bci()); 165 } 166 167 CallStaticJavaNode* call = new CallStaticJavaNode(kit.C, tf(), target, method()); 168 if (is_inlined_method_handle_intrinsic(jvms, method())) { 169 // To be able to issue a direct call and skip a call to MH.linkTo*/invokeBasic adapter, 170 // additional information about the method being invoked should be attached 171 // to the call site to make resolution logic work 172 // (see SharedRuntime::resolve_static_call_C). 173 call->set_override_symbolic_info(true); 174 } 175 _call_node = call; // Save the call node in case we need it later 176 if (!is_static) { 177 // Make an explicit receiver null_check as part of this call. 178 // Since we share a map with the caller, his JVMS gets adjusted. 179 kit.null_check_receiver_before_call(method()); 180 if (kit.stopped()) { 181 // And dump it back to the caller, decorated with any exceptions: 182 return kit.transfer_exceptions_into_jvms(); 183 } 184 // Mark the call node as virtual, sort of: 185 call->set_optimized_virtual(true); 186 if (method()->is_method_handle_intrinsic() || 187 method()->is_compiled_lambda_form()) { 188 call->set_method_handle_invoke(true); 189 } 190 } 191 kit.set_arguments_for_java_call(call, is_late_inline()); 192 if (kit.stopped()) { 193 return kit.transfer_exceptions_into_jvms(); 194 } 195 kit.set_edges_for_java_call(call, false, _separate_io_proj); 196 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj); 197 kit.push_node(method()->return_type()->basic_type(), ret); 198 return kit.transfer_exceptions_into_jvms(); 199 } 200 201 //--------------------------VirtualCallGenerator------------------------------ 202 // Internal class which handles all out-of-line calls checking receiver type. 203 class VirtualCallGenerator : public CallGenerator { 204 private: 205 int _vtable_index; 206 bool _separate_io_proj; 207 CallDynamicJavaNode* _call_node; 208 209 protected: 210 void set_call_node(CallDynamicJavaNode* call) { _call_node = call; } 211 212 public: 213 VirtualCallGenerator(ciMethod* method, int vtable_index, bool separate_io_proj) 214 : CallGenerator(method), _vtable_index(vtable_index), _separate_io_proj(separate_io_proj), _call_node(nullptr) 215 { 216 assert(vtable_index == Method::invalid_vtable_index || 217 vtable_index >= 0, "either invalid or usable"); 218 } 219 virtual bool is_virtual() const { return true; } 220 virtual JVMState* generate(JVMState* jvms); 221 222 virtual CallNode* call_node() const { return _call_node; } 223 int vtable_index() const { return _vtable_index; } 224 225 virtual CallGenerator* with_call_node(CallNode* call) { 226 VirtualCallGenerator* cg = new VirtualCallGenerator(method(), _vtable_index, _separate_io_proj); 227 cg->set_call_node(call->as_CallDynamicJava()); 228 return cg; 229 } 230 }; 231 232 JVMState* VirtualCallGenerator::generate(JVMState* jvms) { 233 GraphKit kit(jvms); 234 Node* receiver = kit.argument(0); 235 kit.C->print_inlining_update(this); 236 237 if (kit.C->log() != nullptr) { 238 kit.C->log()->elem("virtual_call bci='%d'", jvms->bci()); 239 } 240 241 // If the receiver is a constant null, do not torture the system 242 // by attempting to call through it. The compile will proceed 243 // correctly, but may bail out in final_graph_reshaping, because 244 // the call instruction will have a seemingly deficient out-count. 245 // (The bailout says something misleading about an "infinite loop".) 246 if (kit.gvn().type(receiver)->higher_equal(TypePtr::NULL_PTR)) { 247 assert(Bytecodes::is_invoke(kit.java_bc()), "%d: %s", kit.java_bc(), Bytecodes::name(kit.java_bc())); 248 ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci()); 249 int arg_size = declared_method->signature()->arg_size_for_bc(kit.java_bc()); 250 kit.inc_sp(arg_size); // restore arguments 251 kit.uncommon_trap(Deoptimization::Reason_null_check, 252 Deoptimization::Action_none, 253 nullptr, "null receiver"); 254 return kit.transfer_exceptions_into_jvms(); 255 } 256 257 // Ideally we would unconditionally do a null check here and let it 258 // be converted to an implicit check based on profile information. 259 // However currently the conversion to implicit null checks in 260 // Block::implicit_null_check() only looks for loads and stores, not calls. 261 ciMethod *caller = kit.method(); 262 ciMethodData *caller_md = (caller == nullptr) ? nullptr : caller->method_data(); 263 if (!UseInlineCaches || !ImplicitNullChecks || !os::zero_page_read_protected() || 264 ((ImplicitNullCheckThreshold > 0) && caller_md && 265 (caller_md->trap_count(Deoptimization::Reason_null_check) 266 >= (uint)ImplicitNullCheckThreshold))) { 267 // Make an explicit receiver null_check as part of this call. 268 // Since we share a map with the caller, his JVMS gets adjusted. 269 receiver = kit.null_check_receiver_before_call(method()); 270 if (kit.stopped()) { 271 // And dump it back to the caller, decorated with any exceptions: 272 return kit.transfer_exceptions_into_jvms(); 273 } 274 } 275 276 assert(!method()->is_static(), "virtual call must not be to static"); 277 assert(!method()->is_final(), "virtual call should not be to final"); 278 assert(!method()->is_private(), "virtual call should not be to private"); 279 assert(_vtable_index == Method::invalid_vtable_index || !UseInlineCaches, 280 "no vtable calls if +UseInlineCaches "); 281 address target = SharedRuntime::get_resolve_virtual_call_stub(); 282 // Normal inline cache used for call 283 CallDynamicJavaNode* call = new CallDynamicJavaNode(tf(), target, method(), _vtable_index); 284 if (is_inlined_method_handle_intrinsic(jvms, method())) { 285 // To be able to issue a direct call (optimized virtual or virtual) 286 // and skip a call to MH.linkTo*/invokeBasic adapter, additional information 287 // about the method being invoked should be attached to the call site to 288 // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C). 289 call->set_override_symbolic_info(true); 290 } 291 _call_node = call; // Save the call node in case we need it later 292 293 kit.set_arguments_for_java_call(call); 294 if (kit.stopped()) { 295 return kit.transfer_exceptions_into_jvms(); 296 } 297 kit.set_edges_for_java_call(call, false /*must_throw*/, _separate_io_proj); 298 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj); 299 kit.push_node(method()->return_type()->basic_type(), ret); 300 301 // Represent the effect of an implicit receiver null_check 302 // as part of this call. Since we share a map with the caller, 303 // his JVMS gets adjusted. 304 kit.cast_not_null(receiver); 305 return kit.transfer_exceptions_into_jvms(); 306 } 307 308 CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) { 309 if (InlineTree::check_can_parse(m) != nullptr) return nullptr; 310 return new ParseGenerator(m, expected_uses); 311 } 312 313 // As a special case, the JVMS passed to this CallGenerator is 314 // for the method execution already in progress, not just the JVMS 315 // of the caller. Thus, this CallGenerator cannot be mixed with others! 316 CallGenerator* CallGenerator::for_osr(ciMethod* m, int osr_bci) { 317 if (InlineTree::check_can_parse(m) != nullptr) return nullptr; 318 float past_uses = m->interpreter_invocation_count(); 319 float expected_uses = past_uses; 320 return new ParseGenerator(m, expected_uses, true); 321 } 322 323 CallGenerator* CallGenerator::for_direct_call(ciMethod* m, bool separate_io_proj) { 324 assert(!m->is_abstract(), "for_direct_call mismatch"); 325 return new DirectCallGenerator(m, separate_io_proj); 326 } 327 328 CallGenerator* CallGenerator::for_virtual_call(ciMethod* m, int vtable_index) { 329 assert(!m->is_static(), "for_virtual_call mismatch"); 330 assert(!m->is_method_handle_intrinsic(), "should be a direct call"); 331 return new VirtualCallGenerator(m, vtable_index, false /*separate_io_projs*/); 332 } 333 334 // Allow inlining decisions to be delayed 335 class LateInlineCallGenerator : public DirectCallGenerator { 336 private: 337 jlong _unique_id; // unique id for log compilation 338 bool _is_pure_call; // a hint that the call doesn't have important side effects to care about 339 340 protected: 341 CallGenerator* _inline_cg; 342 virtual bool do_late_inline_check(Compile* C, JVMState* jvms) { return true; } 343 virtual CallGenerator* inline_cg() const { return _inline_cg; } 344 virtual bool is_pure_call() const { return _is_pure_call; } 345 346 public: 347 LateInlineCallGenerator(ciMethod* method, CallGenerator* inline_cg, bool is_pure_call = false) : 348 DirectCallGenerator(method, true), _unique_id(0), _is_pure_call(is_pure_call), _inline_cg(inline_cg) {} 349 350 virtual bool is_late_inline() const { return true; } 351 352 // Convert the CallStaticJava into an inline 353 virtual void do_late_inline(); 354 355 virtual JVMState* generate(JVMState* jvms) { 356 Compile *C = Compile::current(); 357 358 C->log_inline_id(this); 359 360 // Record that this call site should be revisited once the main 361 // parse is finished. 362 if (!is_mh_late_inline()) { 363 C->add_late_inline(this); 364 } 365 366 // Emit the CallStaticJava and request separate projections so 367 // that the late inlining logic can distinguish between fall 368 // through and exceptional uses of the memory and io projections 369 // as is done for allocations and macro expansion. 370 return DirectCallGenerator::generate(jvms); 371 } 372 373 virtual void print_inlining_late(InliningResult result, const char* msg) { 374 CallNode* call = call_node(); 375 Compile* C = Compile::current(); 376 C->print_inlining_assert_ready(); 377 C->print_inlining(method(), call->jvms()->depth()-1, call->jvms()->bci(), result, msg); 378 C->print_inlining_move_to(this); 379 C->print_inlining_update_delayed(this); 380 } 381 382 virtual void set_unique_id(jlong id) { 383 _unique_id = id; 384 } 385 386 virtual jlong unique_id() const { 387 return _unique_id; 388 } 389 390 virtual CallGenerator* inline_cg() { 391 return _inline_cg; 392 } 393 394 virtual CallGenerator* with_call_node(CallNode* call) { 395 LateInlineCallGenerator* cg = new LateInlineCallGenerator(method(), _inline_cg, _is_pure_call); 396 cg->set_call_node(call->as_CallStaticJava()); 397 return cg; 398 } 399 }; 400 401 CallGenerator* CallGenerator::for_late_inline(ciMethod* method, CallGenerator* inline_cg) { 402 return new LateInlineCallGenerator(method, inline_cg); 403 } 404 405 class LateInlineMHCallGenerator : public LateInlineCallGenerator { 406 ciMethod* _caller; 407 bool _input_not_const; 408 409 virtual bool do_late_inline_check(Compile* C, JVMState* jvms); 410 411 public: 412 LateInlineMHCallGenerator(ciMethod* caller, ciMethod* callee, bool input_not_const) : 413 LateInlineCallGenerator(callee, nullptr), _caller(caller), _input_not_const(input_not_const) {} 414 415 virtual bool is_mh_late_inline() const { return true; } 416 417 // Convert the CallStaticJava into an inline 418 virtual void do_late_inline(); 419 420 virtual JVMState* generate(JVMState* jvms) { 421 JVMState* new_jvms = LateInlineCallGenerator::generate(jvms); 422 423 Compile* C = Compile::current(); 424 if (_input_not_const) { 425 // inlining won't be possible so no need to enqueue right now. 426 call_node()->set_generator(this); 427 } else { 428 C->add_late_inline(this); 429 } 430 return new_jvms; 431 } 432 433 virtual CallGenerator* with_call_node(CallNode* call) { 434 LateInlineMHCallGenerator* cg = new LateInlineMHCallGenerator(_caller, method(), _input_not_const); 435 cg->set_call_node(call->as_CallStaticJava()); 436 return cg; 437 } 438 }; 439 440 bool LateInlineMHCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) { 441 // When inlining a virtual call, the null check at the call and the call itself can throw. These 2 paths have different 442 // expression stacks which causes late inlining to break. The MH invoker is not expected to be called from a method with 443 // exception handlers. When there is no exception handler, GraphKit::builtin_throw() pops the stack which solves the issue 444 // of late inlining with exceptions. 445 assert(!jvms->method()->has_exception_handlers() || 446 (method()->intrinsic_id() != vmIntrinsics::_linkToVirtual && 447 method()->intrinsic_id() != vmIntrinsics::_linkToInterface), "no exception handler expected"); 448 // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call. 449 bool allow_inline = C->inlining_incrementally(); 450 bool input_not_const = true; 451 CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), allow_inline, input_not_const); 452 assert(!input_not_const, "sanity"); // shouldn't have been scheduled for inlining in the first place 453 454 if (cg != nullptr) { 455 // AlwaysIncrementalInline causes for_method_handle_inline() to 456 // return a LateInlineCallGenerator. Extract the 457 // InlineCallGenerator from it. 458 if (AlwaysIncrementalInline && cg->is_late_inline() && !cg->is_virtual_late_inline()) { 459 cg = cg->inline_cg(); 460 assert(cg != nullptr, "inline call generator expected"); 461 } 462 463 if (!allow_inline && (C->print_inlining() || C->print_intrinsics())) { 464 C->print_inlining(cg->method(), jvms->depth()-1, call_node()->jvms()->bci(), InliningResult::FAILURE, 465 "late method handle call resolution"); 466 } 467 assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline || StressIncrementalInlining, "we're doing late inlining"); 468 _inline_cg = cg; 469 C->dec_number_of_mh_late_inlines(); 470 return true; 471 } else { 472 // Method handle call which has a constant appendix argument should be either inlined or replaced with a direct call 473 // unless there's a signature mismatch between caller and callee. If the failure occurs, there's not much to be improved later, 474 // so don't reinstall the generator to avoid pushing the generator between IGVN and incremental inlining indefinitely. 475 return false; 476 } 477 } 478 479 CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const) { 480 assert(IncrementalInlineMH, "required"); 481 Compile::current()->inc_number_of_mh_late_inlines(); 482 CallGenerator* cg = new LateInlineMHCallGenerator(caller, callee, input_not_const); 483 return cg; 484 } 485 486 // Allow inlining decisions to be delayed 487 class LateInlineVirtualCallGenerator : public VirtualCallGenerator { 488 private: 489 jlong _unique_id; // unique id for log compilation 490 CallGenerator* _inline_cg; 491 ciMethod* _callee; 492 bool _is_pure_call; 493 float _prof_factor; 494 495 protected: 496 virtual bool do_late_inline_check(Compile* C, JVMState* jvms); 497 virtual CallGenerator* inline_cg() const { return _inline_cg; } 498 virtual bool is_pure_call() const { return _is_pure_call; } 499 500 public: 501 LateInlineVirtualCallGenerator(ciMethod* method, int vtable_index, float prof_factor) 502 : VirtualCallGenerator(method, vtable_index, true /*separate_io_projs*/), 503 _unique_id(0), _inline_cg(nullptr), _callee(nullptr), _is_pure_call(false), _prof_factor(prof_factor) { 504 assert(IncrementalInlineVirtual, "required"); 505 } 506 507 virtual bool is_late_inline() const { return true; } 508 509 virtual bool is_virtual_late_inline() const { return true; } 510 511 // Convert the CallDynamicJava into an inline 512 virtual void do_late_inline(); 513 514 virtual void set_callee_method(ciMethod* m) { 515 assert(_callee == nullptr, "repeated inlining attempt"); 516 _callee = m; 517 } 518 519 virtual JVMState* generate(JVMState* jvms) { 520 // Emit the CallDynamicJava and request separate projections so 521 // that the late inlining logic can distinguish between fall 522 // through and exceptional uses of the memory and io projections 523 // as is done for allocations and macro expansion. 524 JVMState* new_jvms = VirtualCallGenerator::generate(jvms); 525 if (call_node() != nullptr) { 526 call_node()->set_generator(this); 527 } 528 return new_jvms; 529 } 530 531 virtual void print_inlining_late(InliningResult result, const char* msg) { 532 CallNode* call = call_node(); 533 Compile* C = Compile::current(); 534 C->print_inlining_assert_ready(); 535 C->print_inlining(method(), call->jvms()->depth()-1, call->jvms()->bci(), result, msg); 536 C->print_inlining_move_to(this); 537 C->print_inlining_update_delayed(this); 538 } 539 540 virtual void set_unique_id(jlong id) { 541 _unique_id = id; 542 } 543 544 virtual jlong unique_id() const { 545 return _unique_id; 546 } 547 548 virtual CallGenerator* with_call_node(CallNode* call) { 549 LateInlineVirtualCallGenerator* cg = new LateInlineVirtualCallGenerator(method(), vtable_index(), _prof_factor); 550 cg->set_call_node(call->as_CallDynamicJava()); 551 return cg; 552 } 553 }; 554 555 bool LateInlineVirtualCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) { 556 // Method handle linker case is handled in CallDynamicJavaNode::Ideal(). 557 // Unless inlining is performed, _override_symbolic_info bit will be set in DirectCallGenerator::generate(). 558 559 // Implicit receiver null checks introduce problems when exception states are combined. 560 Node* receiver = jvms->map()->argument(jvms, 0); 561 const Type* recv_type = C->initial_gvn()->type(receiver); 562 if (recv_type->maybe_null()) { 563 if (C->print_inlining() || C->print_intrinsics()) { 564 C->print_inlining(method(), jvms->depth()-1, call_node()->jvms()->bci(), InliningResult::FAILURE, 565 "late call devirtualization failed (receiver may be null)"); 566 } 567 return false; 568 } 569 // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call. 570 bool allow_inline = C->inlining_incrementally(); 571 if (!allow_inline && _callee->holder()->is_interface()) { 572 // Don't convert the interface call to a direct call guarded by an interface subtype check. 573 if (C->print_inlining() || C->print_intrinsics()) { 574 C->print_inlining(method(), jvms->depth()-1, call_node()->jvms()->bci(), InliningResult::FAILURE, 575 "late call devirtualization failed (interface call)"); 576 } 577 return false; 578 } 579 CallGenerator* cg = C->call_generator(_callee, 580 vtable_index(), 581 false /*call_does_dispatch*/, 582 jvms, 583 allow_inline, 584 _prof_factor, 585 nullptr /*speculative_receiver_type*/, 586 true /*allow_intrinsics*/); 587 588 if (cg != nullptr) { 589 if (!allow_inline && (C->print_inlining() || C->print_intrinsics())) { 590 C->print_inlining(cg->method(), jvms->depth()-1, call_node()->jvms()->bci(), InliningResult::FAILURE, 591 "late call devirtualization"); 592 } 593 assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline || StressIncrementalInlining, "we're doing late inlining"); 594 _inline_cg = cg; 595 return true; 596 } else { 597 // Virtual call which provably doesn't dispatch should be either inlined or replaced with a direct call. 598 assert(false, "no progress"); 599 return false; 600 } 601 } 602 603 CallGenerator* CallGenerator::for_late_inline_virtual(ciMethod* m, int vtable_index, float prof_factor) { 604 assert(IncrementalInlineVirtual, "required"); 605 assert(!m->is_static(), "for_virtual_call mismatch"); 606 assert(!m->is_method_handle_intrinsic(), "should be a direct call"); 607 return new LateInlineVirtualCallGenerator(m, vtable_index, prof_factor); 608 } 609 610 void LateInlineCallGenerator::do_late_inline() { 611 CallGenerator::do_late_inline_helper(); 612 } 613 614 void LateInlineMHCallGenerator::do_late_inline() { 615 CallGenerator::do_late_inline_helper(); 616 } 617 618 void LateInlineVirtualCallGenerator::do_late_inline() { 619 assert(_callee != nullptr, "required"); // set up in CallDynamicJavaNode::Ideal 620 CallGenerator::do_late_inline_helper(); 621 } 622 623 void CallGenerator::do_late_inline_helper() { 624 assert(is_late_inline(), "only late inline allowed"); 625 626 // Can't inline it 627 CallNode* call = call_node(); 628 if (call == nullptr || call->outcnt() == 0 || 629 call->in(0) == nullptr || call->in(0)->is_top()) { 630 return; 631 } 632 633 const TypeTuple* r = call->tf()->domain_cc(); 634 for (uint i1 = TypeFunc::Parms; i1 < r->cnt(); i1++) { 635 if (call->in(i1)->is_top() && r->field_at(i1) != Type::HALF) { 636 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing"); 637 return; 638 } 639 } 640 641 if (call->in(TypeFunc::Memory)->is_top()) { 642 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing"); 643 return; 644 } 645 if (call->in(TypeFunc::Memory)->is_MergeMem()) { 646 MergeMemNode* merge_mem = call->in(TypeFunc::Memory)->as_MergeMem(); 647 if (merge_mem->base_memory() == merge_mem->empty_memory()) { 648 return; // dead path 649 } 650 } 651 652 // check for unreachable loop 653 CallProjections* callprojs = call->extract_projections(true); 654 if ((callprojs->fallthrough_catchproj == call->in(0)) || 655 (callprojs->catchall_catchproj == call->in(0)) || 656 (callprojs->fallthrough_memproj == call->in(TypeFunc::Memory)) || 657 (callprojs->catchall_memproj == call->in(TypeFunc::Memory)) || 658 (callprojs->fallthrough_ioproj == call->in(TypeFunc::I_O)) || 659 (callprojs->catchall_ioproj == call->in(TypeFunc::I_O)) || 660 (callprojs->exobj != nullptr && call->find_edge(callprojs->exobj) != -1)) { 661 return; 662 } 663 664 Compile* C = Compile::current(); 665 // Remove inlined methods from Compiler's lists. 666 if (call->is_macro()) { 667 C->remove_macro_node(call); 668 } 669 670 671 bool result_not_used = true; 672 for (uint i = 0; i < callprojs->nb_resproj; i++) { 673 if (callprojs->resproj[i] != nullptr) { 674 if (callprojs->resproj[i]->outcnt() != 0) { 675 result_not_used = false; 676 } 677 if (call->find_edge(callprojs->resproj[i]) != -1) { 678 return; 679 } 680 } 681 } 682 683 if (is_pure_call() && result_not_used) { 684 // The call is marked as pure (no important side effects), but result isn't used. 685 // It's safe to remove the call. 686 GraphKit kit(call->jvms()); 687 kit.replace_call(call, C->top(), true); 688 } else { 689 // Make a clone of the JVMState that appropriate to use for driving a parse 690 JVMState* old_jvms = call->jvms(); 691 JVMState* jvms = old_jvms->clone_shallow(C); 692 uint size = call->req(); 693 SafePointNode* map = new SafePointNode(size, jvms); 694 for (uint i1 = 0; i1 < size; i1++) { 695 map->init_req(i1, call->in(i1)); 696 } 697 698 PhaseGVN& gvn = *C->initial_gvn(); 699 // Make sure the state is a MergeMem for parsing. 700 if (!map->in(TypeFunc::Memory)->is_MergeMem()) { 701 Node* mem = MergeMemNode::make(map->in(TypeFunc::Memory)); 702 gvn.set_type_bottom(mem); 703 map->set_req(TypeFunc::Memory, mem); 704 } 705 706 // blow away old call arguments 707 for (uint i1 = TypeFunc::Parms; i1 < r->cnt(); i1++) { 708 map->set_req(i1, C->top()); 709 } 710 jvms->set_map(map); 711 712 // Make enough space in the expression stack to transfer 713 // the incoming arguments and return value. 714 map->ensure_stack(jvms, jvms->method()->max_stack()); 715 const TypeTuple* domain_sig = call->_tf->domain_sig(); 716 uint nargs = method()->arg_size(); 717 assert(domain_sig->cnt() - TypeFunc::Parms == nargs, "inconsistent signature"); 718 719 uint j = TypeFunc::Parms; 720 int arg_num = 0; 721 for (uint i1 = 0; i1 < nargs; i1++) { 722 const Type* t = domain_sig->field_at(TypeFunc::Parms + i1); 723 if (t->is_inlinetypeptr() && !method()->get_Method()->mismatch() && method()->is_scalarized_arg(arg_num)) { 724 // Inline type arguments are not passed by reference: we get an argument per 725 // field of the inline type. Build InlineTypeNodes from the inline type arguments. 726 GraphKit arg_kit(jvms, &gvn); 727 Node* vt = InlineTypeNode::make_from_multi(&arg_kit, call, t->inline_klass(), j, /* in= */ true, /* null_free= */ !t->maybe_null()); 728 map->set_control(arg_kit.control()); 729 map->set_argument(jvms, i1, vt); 730 } else { 731 map->set_argument(jvms, i1, call->in(j++)); 732 } 733 if (t != Type::HALF) { 734 arg_num++; 735 } 736 } 737 738 C->print_inlining_assert_ready(); 739 740 C->print_inlining_move_to(this); 741 742 C->log_late_inline(this); 743 744 // JVMState is ready, so time to perform some checks and prepare for inlining attempt. 745 if (!do_late_inline_check(C, jvms)) { 746 map->disconnect_inputs(C); 747 C->print_inlining_update_delayed(this); 748 return; 749 } 750 if (C->print_inlining() && (is_mh_late_inline() || is_virtual_late_inline())) { 751 C->print_inlining_update_delayed(this); 752 } 753 754 // Check if we are late inlining a method handle call that returns an inline type as fields. 755 Node* buffer_oop = nullptr; 756 ciMethod* inline_method = inline_cg()->method(); 757 ciType* return_type = inline_method->return_type(); 758 if (!call->tf()->returns_inline_type_as_fields() && is_mh_late_inline() && 759 return_type->is_inlinetype() && return_type->as_inline_klass()->can_be_returned_as_fields()) { 760 // Allocate a buffer for the inline type returned as fields because the caller expects an oop return. 761 // Do this before the method handle call in case the buffer allocation triggers deoptimization and 762 // we need to "re-execute" the call in the interpreter (to make sure the call is only executed once). 763 GraphKit arg_kit(jvms, &gvn); 764 { 765 PreserveReexecuteState preexecs(&arg_kit); 766 arg_kit.jvms()->set_should_reexecute(true); 767 arg_kit.inc_sp(nargs); 768 Node* klass_node = arg_kit.makecon(TypeKlassPtr::make(return_type->as_inline_klass())); 769 buffer_oop = arg_kit.new_instance(klass_node, nullptr, nullptr, /* deoptimize_on_exception */ true); 770 } 771 jvms = arg_kit.transfer_exceptions_into_jvms(); 772 } 773 774 // Setup default node notes to be picked up by the inlining 775 Node_Notes* old_nn = C->node_notes_at(call->_idx); 776 if (old_nn != nullptr) { 777 Node_Notes* entry_nn = old_nn->clone(C); 778 entry_nn->set_jvms(jvms); 779 C->set_default_node_notes(entry_nn); 780 } 781 782 // Now perform the inlining using the synthesized JVMState 783 JVMState* new_jvms = inline_cg()->generate(jvms); 784 if (new_jvms == nullptr) return; // no change 785 if (C->failing()) return; 786 787 // Capture any exceptional control flow 788 GraphKit kit(new_jvms); 789 790 // Find the result object 791 Node* result = C->top(); 792 int result_size = method()->return_type()->size(); 793 if (result_size != 0 && !kit.stopped()) { 794 result = (result_size == 1) ? kit.pop() : kit.pop_pair(); 795 } 796 797 if (call->is_CallStaticJava() && call->as_CallStaticJava()->is_boxing_method()) { 798 result = kit.must_be_not_null(result, false); 799 } 800 801 if (inline_cg()->is_inline()) { 802 C->set_has_loops(C->has_loops() || inline_method->has_loops()); 803 C->env()->notice_inlined_method(inline_method); 804 } 805 C->set_inlining_progress(true); 806 C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup 807 808 // Handle inline type returns 809 InlineTypeNode* vt = result->isa_InlineType(); 810 if (vt != nullptr) { 811 if (call->tf()->returns_inline_type_as_fields()) { 812 vt->replace_call_results(&kit, call, C); 813 } else if (vt->is_InlineType()) { 814 // Result might still be allocated (for example, if it has been stored to a non-flat field) 815 if (!vt->is_allocated(&kit.gvn())) { 816 assert(buffer_oop != nullptr, "should have allocated a buffer"); 817 RegionNode* region = new RegionNode(3); 818 819 // Check if result is null 820 Node* null_ctl = kit.top(); 821 kit.null_check_common(vt->get_is_init(), T_INT, false, &null_ctl); 822 region->init_req(1, null_ctl); 823 PhiNode* oop = PhiNode::make(region, kit.gvn().zerocon(T_OBJECT), TypeInstPtr::make(TypePtr::BotPTR, vt->type()->inline_klass())); 824 Node* init_mem = kit.reset_memory(); 825 PhiNode* mem = PhiNode::make(region, init_mem, Type::MEMORY, TypePtr::BOTTOM); 826 827 // Not null, initialize the buffer 828 kit.set_all_memory(init_mem); 829 vt->store(&kit, buffer_oop, buffer_oop, vt->type()->inline_klass()); 830 // Do not let stores that initialize this buffer be reordered with a subsequent 831 // store that would make this buffer accessible by other threads. 832 AllocateNode* alloc = AllocateNode::Ideal_allocation(buffer_oop); 833 assert(alloc != nullptr, "must have an allocation node"); 834 kit.insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 835 region->init_req(2, kit.control()); 836 oop->init_req(2, buffer_oop); 837 mem->init_req(2, kit.merged_memory()); 838 839 // Update oop input to buffer 840 kit.gvn().hash_delete(vt); 841 vt->set_oop(kit.gvn(), kit.gvn().transform(oop)); 842 vt->set_is_buffered(kit.gvn()); 843 vt = kit.gvn().transform(vt)->as_InlineType(); 844 845 kit.set_control(kit.gvn().transform(region)); 846 kit.set_all_memory(kit.gvn().transform(mem)); 847 kit.record_for_igvn(region); 848 kit.record_for_igvn(oop); 849 kit.record_for_igvn(mem); 850 } 851 result = vt; 852 } 853 DEBUG_ONLY(buffer_oop = nullptr); 854 } else { 855 assert(result->is_top() || !call->tf()->returns_inline_type_as_fields(), "Unexpected return value"); 856 } 857 assert(buffer_oop == nullptr, "unused buffer allocation"); 858 859 kit.replace_call(call, result, true); 860 } 861 } 862 863 class LateInlineStringCallGenerator : public LateInlineCallGenerator { 864 865 public: 866 LateInlineStringCallGenerator(ciMethod* method, CallGenerator* inline_cg) : 867 LateInlineCallGenerator(method, inline_cg) {} 868 869 virtual JVMState* generate(JVMState* jvms) { 870 Compile *C = Compile::current(); 871 872 C->log_inline_id(this); 873 874 C->add_string_late_inline(this); 875 876 JVMState* new_jvms = DirectCallGenerator::generate(jvms); 877 return new_jvms; 878 } 879 880 virtual bool is_string_late_inline() const { return true; } 881 882 virtual CallGenerator* with_call_node(CallNode* call) { 883 LateInlineStringCallGenerator* cg = new LateInlineStringCallGenerator(method(), _inline_cg); 884 cg->set_call_node(call->as_CallStaticJava()); 885 return cg; 886 } 887 }; 888 889 CallGenerator* CallGenerator::for_string_late_inline(ciMethod* method, CallGenerator* inline_cg) { 890 return new LateInlineStringCallGenerator(method, inline_cg); 891 } 892 893 class LateInlineBoxingCallGenerator : public LateInlineCallGenerator { 894 895 public: 896 LateInlineBoxingCallGenerator(ciMethod* method, CallGenerator* inline_cg) : 897 LateInlineCallGenerator(method, inline_cg, /*is_pure=*/true) {} 898 899 virtual JVMState* generate(JVMState* jvms) { 900 Compile *C = Compile::current(); 901 902 C->log_inline_id(this); 903 904 C->add_boxing_late_inline(this); 905 906 JVMState* new_jvms = DirectCallGenerator::generate(jvms); 907 return new_jvms; 908 } 909 910 virtual CallGenerator* with_call_node(CallNode* call) { 911 LateInlineBoxingCallGenerator* cg = new LateInlineBoxingCallGenerator(method(), _inline_cg); 912 cg->set_call_node(call->as_CallStaticJava()); 913 return cg; 914 } 915 }; 916 917 CallGenerator* CallGenerator::for_boxing_late_inline(ciMethod* method, CallGenerator* inline_cg) { 918 return new LateInlineBoxingCallGenerator(method, inline_cg); 919 } 920 921 class LateInlineVectorReboxingCallGenerator : public LateInlineCallGenerator { 922 923 public: 924 LateInlineVectorReboxingCallGenerator(ciMethod* method, CallGenerator* inline_cg) : 925 LateInlineCallGenerator(method, inline_cg, /*is_pure=*/true) {} 926 927 virtual JVMState* generate(JVMState* jvms) { 928 Compile *C = Compile::current(); 929 930 C->log_inline_id(this); 931 932 C->add_vector_reboxing_late_inline(this); 933 934 JVMState* new_jvms = DirectCallGenerator::generate(jvms); 935 return new_jvms; 936 } 937 938 virtual CallGenerator* with_call_node(CallNode* call) { 939 LateInlineVectorReboxingCallGenerator* cg = new LateInlineVectorReboxingCallGenerator(method(), _inline_cg); 940 cg->set_call_node(call->as_CallStaticJava()); 941 return cg; 942 } 943 }; 944 945 // static CallGenerator* for_vector_reboxing_late_inline(ciMethod* m, CallGenerator* inline_cg); 946 CallGenerator* CallGenerator::for_vector_reboxing_late_inline(ciMethod* method, CallGenerator* inline_cg) { 947 return new LateInlineVectorReboxingCallGenerator(method, inline_cg); 948 } 949 950 //------------------------PredictedCallGenerator------------------------------ 951 // Internal class which handles all out-of-line calls checking receiver type. 952 class PredictedCallGenerator : public CallGenerator { 953 ciKlass* _predicted_receiver; 954 CallGenerator* _if_missed; 955 CallGenerator* _if_hit; 956 float _hit_prob; 957 bool _exact_check; 958 959 public: 960 PredictedCallGenerator(ciKlass* predicted_receiver, 961 CallGenerator* if_missed, 962 CallGenerator* if_hit, bool exact_check, 963 float hit_prob) 964 : CallGenerator(if_missed->method()) 965 { 966 // The call profile data may predict the hit_prob as extreme as 0 or 1. 967 // Remove the extremes values from the range. 968 if (hit_prob > PROB_MAX) hit_prob = PROB_MAX; 969 if (hit_prob < PROB_MIN) hit_prob = PROB_MIN; 970 971 _predicted_receiver = predicted_receiver; 972 _if_missed = if_missed; 973 _if_hit = if_hit; 974 _hit_prob = hit_prob; 975 _exact_check = exact_check; 976 } 977 978 virtual bool is_virtual() const { return true; } 979 virtual bool is_inline() const { return _if_hit->is_inline(); } 980 virtual bool is_deferred() const { return _if_hit->is_deferred(); } 981 982 virtual JVMState* generate(JVMState* jvms); 983 }; 984 985 986 CallGenerator* CallGenerator::for_predicted_call(ciKlass* predicted_receiver, 987 CallGenerator* if_missed, 988 CallGenerator* if_hit, 989 float hit_prob) { 990 return new PredictedCallGenerator(predicted_receiver, if_missed, if_hit, 991 /*exact_check=*/true, hit_prob); 992 } 993 994 CallGenerator* CallGenerator::for_guarded_call(ciKlass* guarded_receiver, 995 CallGenerator* if_missed, 996 CallGenerator* if_hit) { 997 return new PredictedCallGenerator(guarded_receiver, if_missed, if_hit, 998 /*exact_check=*/false, PROB_ALWAYS); 999 } 1000 1001 JVMState* PredictedCallGenerator::generate(JVMState* jvms) { 1002 GraphKit kit(jvms); 1003 kit.C->print_inlining_update(this); 1004 PhaseGVN& gvn = kit.gvn(); 1005 // We need an explicit receiver null_check before checking its type. 1006 // We share a map with the caller, so his JVMS gets adjusted. 1007 Node* receiver = kit.argument(0); 1008 CompileLog* log = kit.C->log(); 1009 if (log != nullptr) { 1010 log->elem("predicted_call bci='%d' exact='%d' klass='%d'", 1011 jvms->bci(), (_exact_check ? 1 : 0), log->identify(_predicted_receiver)); 1012 } 1013 1014 receiver = kit.null_check_receiver_before_call(method()); 1015 if (kit.stopped()) { 1016 return kit.transfer_exceptions_into_jvms(); 1017 } 1018 1019 // Make a copy of the replaced nodes in case we need to restore them 1020 ReplacedNodes replaced_nodes = kit.map()->replaced_nodes(); 1021 replaced_nodes.clone(); 1022 1023 Node* casted_receiver = receiver; // will get updated in place... 1024 Node* slow_ctl = nullptr; 1025 if (_exact_check) { 1026 slow_ctl = kit.type_check_receiver(receiver, _predicted_receiver, _hit_prob, 1027 &casted_receiver); 1028 } else { 1029 slow_ctl = kit.subtype_check_receiver(receiver, _predicted_receiver, 1030 &casted_receiver); 1031 } 1032 1033 SafePointNode* slow_map = nullptr; 1034 JVMState* slow_jvms = nullptr; 1035 { PreserveJVMState pjvms(&kit); 1036 kit.set_control(slow_ctl); 1037 if (!kit.stopped()) { 1038 slow_jvms = _if_missed->generate(kit.sync_jvms()); 1039 if (kit.failing()) 1040 return nullptr; // might happen because of NodeCountInliningCutoff 1041 assert(slow_jvms != nullptr, "must be"); 1042 kit.add_exception_states_from(slow_jvms); 1043 kit.set_map(slow_jvms->map()); 1044 if (!kit.stopped()) 1045 slow_map = kit.stop(); 1046 } 1047 } 1048 1049 if (kit.stopped()) { 1050 // Instance does not match the predicted type. 1051 kit.set_jvms(slow_jvms); 1052 return kit.transfer_exceptions_into_jvms(); 1053 } 1054 1055 // Fall through if the instance matches the desired type. 1056 kit.replace_in_map(receiver, casted_receiver); 1057 1058 // Make the hot call: 1059 JVMState* new_jvms = _if_hit->generate(kit.sync_jvms()); 1060 if (new_jvms == nullptr) { 1061 // Inline failed, so make a direct call. 1062 assert(_if_hit->is_inline(), "must have been a failed inline"); 1063 CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method()); 1064 new_jvms = cg->generate(kit.sync_jvms()); 1065 } 1066 kit.add_exception_states_from(new_jvms); 1067 kit.set_jvms(new_jvms); 1068 1069 // Need to merge slow and fast? 1070 if (slow_map == nullptr) { 1071 // The fast path is the only path remaining. 1072 return kit.transfer_exceptions_into_jvms(); 1073 } 1074 1075 if (kit.stopped()) { 1076 // Inlined method threw an exception, so it's just the slow path after all. 1077 kit.set_jvms(slow_jvms); 1078 return kit.transfer_exceptions_into_jvms(); 1079 } 1080 1081 // Allocate inline types if they are merged with objects (similar to Parse::merge_common()) 1082 uint tos = kit.jvms()->stkoff() + kit.sp(); 1083 uint limit = slow_map->req(); 1084 for (uint i = TypeFunc::Parms; i < limit; i++) { 1085 Node* m = kit.map()->in(i); 1086 Node* n = slow_map->in(i); 1087 const Type* t = gvn.type(m)->meet_speculative(gvn.type(n)); 1088 // TODO 8284443 still needed? 1089 if (m->is_InlineType() && !t->is_inlinetypeptr()) { 1090 // Allocate inline type in fast path 1091 m = m->as_InlineType()->buffer(&kit); 1092 kit.map()->set_req(i, m); 1093 } 1094 if (n->is_InlineType() && !t->is_inlinetypeptr()) { 1095 // Allocate inline type in slow path 1096 PreserveJVMState pjvms(&kit); 1097 kit.set_map(slow_map); 1098 n = n->as_InlineType()->buffer(&kit); 1099 kit.map()->set_req(i, n); 1100 slow_map = kit.stop(); 1101 } 1102 } 1103 1104 // There are 2 branches and the replaced nodes are only valid on 1105 // one: restore the replaced nodes to what they were before the 1106 // branch. 1107 kit.map()->set_replaced_nodes(replaced_nodes); 1108 1109 // Finish the diamond. 1110 kit.C->set_has_split_ifs(true); // Has chance for split-if optimization 1111 RegionNode* region = new RegionNode(3); 1112 region->init_req(1, kit.control()); 1113 region->init_req(2, slow_map->control()); 1114 kit.set_control(gvn.transform(region)); 1115 Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO); 1116 iophi->set_req(2, slow_map->i_o()); 1117 kit.set_i_o(gvn.transform(iophi)); 1118 // Merge memory 1119 kit.merge_memory(slow_map->merged_memory(), region, 2); 1120 // Transform new memory Phis. 1121 for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) { 1122 Node* phi = mms.memory(); 1123 if (phi->is_Phi() && phi->in(0) == region) { 1124 mms.set_memory(gvn.transform(phi)); 1125 } 1126 } 1127 for (uint i = TypeFunc::Parms; i < limit; i++) { 1128 // Skip unused stack slots; fast forward to monoff(); 1129 if (i == tos) { 1130 i = kit.jvms()->monoff(); 1131 if( i >= limit ) break; 1132 } 1133 Node* m = kit.map()->in(i); 1134 Node* n = slow_map->in(i); 1135 if (m != n) { 1136 const Type* t = gvn.type(m)->meet_speculative(gvn.type(n)); 1137 Node* phi = PhiNode::make(region, m, t); 1138 phi->set_req(2, n); 1139 kit.map()->set_req(i, gvn.transform(phi)); 1140 } 1141 } 1142 return kit.transfer_exceptions_into_jvms(); 1143 } 1144 1145 1146 CallGenerator* CallGenerator::for_method_handle_call(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline) { 1147 assert(callee->is_method_handle_intrinsic(), "for_method_handle_call mismatch"); 1148 bool input_not_const; 1149 CallGenerator* cg = CallGenerator::for_method_handle_inline(jvms, caller, callee, allow_inline, input_not_const); 1150 Compile* C = Compile::current(); 1151 bool should_delay = C->should_delay_inlining(); 1152 if (cg != nullptr) { 1153 if (should_delay) { 1154 return CallGenerator::for_late_inline(callee, cg); 1155 } else { 1156 return cg; 1157 } 1158 } 1159 int bci = jvms->bci(); 1160 ciCallProfile profile = caller->call_profile_at_bci(bci); 1161 int call_site_count = caller->scale_count(profile.count()); 1162 1163 if (IncrementalInlineMH && (AlwaysIncrementalInline || 1164 (call_site_count > 0 && (should_delay || input_not_const || !C->inlining_incrementally() || C->over_inlining_cutoff())))) { 1165 return CallGenerator::for_mh_late_inline(caller, callee, input_not_const); 1166 } else { 1167 // Out-of-line call. 1168 return CallGenerator::for_direct_call(callee); 1169 } 1170 } 1171 1172 1173 CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline, bool& input_not_const) { 1174 GraphKit kit(jvms); 1175 PhaseGVN& gvn = kit.gvn(); 1176 Compile* C = kit.C; 1177 vmIntrinsics::ID iid = callee->intrinsic_id(); 1178 input_not_const = true; 1179 if (StressMethodHandleLinkerInlining) { 1180 allow_inline = false; 1181 } 1182 switch (iid) { 1183 case vmIntrinsics::_invokeBasic: 1184 { 1185 // Get MethodHandle receiver: 1186 Node* receiver = kit.argument(0); 1187 if (receiver->Opcode() == Op_ConP) { 1188 input_not_const = false; 1189 const TypeOopPtr* recv_toop = receiver->bottom_type()->isa_oopptr(); 1190 if (recv_toop != nullptr) { 1191 ciMethod* target = recv_toop->const_oop()->as_method_handle()->get_vmtarget(); 1192 const int vtable_index = Method::invalid_vtable_index; 1193 1194 if (!ciMethod::is_consistent_info(callee, target)) { 1195 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), 1196 "signatures mismatch"); 1197 return nullptr; 1198 } 1199 1200 CallGenerator *cg = C->call_generator(target, vtable_index, 1201 false /* call_does_dispatch */, 1202 jvms, 1203 allow_inline, 1204 PROB_ALWAYS); 1205 return cg; 1206 } else { 1207 assert(receiver->bottom_type() == TypePtr::NULL_PTR, "not a null: %s", 1208 Type::str(receiver->bottom_type())); 1209 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), 1210 "receiver is always null"); 1211 } 1212 } else { 1213 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), 1214 "receiver not constant"); 1215 } 1216 } 1217 break; 1218 1219 case vmIntrinsics::_linkToVirtual: 1220 case vmIntrinsics::_linkToStatic: 1221 case vmIntrinsics::_linkToSpecial: 1222 case vmIntrinsics::_linkToInterface: 1223 { 1224 int nargs = callee->arg_size(); 1225 // Get MemberName argument: 1226 Node* member_name = kit.argument(nargs - 1); 1227 if (member_name->Opcode() == Op_ConP) { 1228 input_not_const = false; 1229 const TypeOopPtr* oop_ptr = member_name->bottom_type()->is_oopptr(); 1230 ciMethod* target = oop_ptr->const_oop()->as_member_name()->get_vmtarget(); 1231 1232 if (!ciMethod::is_consistent_info(callee, target)) { 1233 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), 1234 "signatures mismatch"); 1235 return nullptr; 1236 } 1237 1238 // In lambda forms we erase signature types to avoid resolving issues 1239 // involving class loaders. When we optimize a method handle invoke 1240 // to a direct call we must cast the receiver and arguments to its 1241 // actual types. 1242 ciSignature* signature = target->signature(); 1243 const int receiver_skip = target->is_static() ? 0 : 1; 1244 // Cast receiver to its type. 1245 if (!target->is_static()) { 1246 Node* recv = kit.argument(0); 1247 Node* casted_recv = kit.maybe_narrow_object_type(recv, signature->accessing_klass()); 1248 if (casted_recv->is_top()) { 1249 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), 1250 "argument types mismatch"); 1251 return nullptr; // FIXME: effectively dead; issue a halt node instead 1252 } else if (casted_recv != recv) { 1253 kit.set_argument(0, casted_recv); 1254 } 1255 } 1256 // Cast reference arguments to its type. 1257 for (int i = 0, j = 0; i < signature->count(); i++) { 1258 ciType* t = signature->type_at(i); 1259 if (t->is_klass()) { 1260 Node* arg = kit.argument(receiver_skip + j); 1261 Node* casted_arg = kit.maybe_narrow_object_type(arg, t->as_klass()); 1262 if (casted_arg->is_top()) { 1263 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), 1264 "argument types mismatch"); 1265 return nullptr; // FIXME: effectively dead; issue a halt node instead 1266 } else if (casted_arg != arg) { 1267 kit.set_argument(receiver_skip + j, casted_arg); 1268 } 1269 } 1270 j += t->size(); // long and double take two slots 1271 } 1272 1273 // Try to get the most accurate receiver type 1274 const bool is_virtual = (iid == vmIntrinsics::_linkToVirtual); 1275 const bool is_virtual_or_interface = (is_virtual || iid == vmIntrinsics::_linkToInterface); 1276 int vtable_index = Method::invalid_vtable_index; 1277 bool call_does_dispatch = false; 1278 1279 ciKlass* speculative_receiver_type = nullptr; 1280 if (is_virtual_or_interface) { 1281 ciInstanceKlass* klass = target->holder(); 1282 Node* receiver_node = kit.argument(0); 1283 const TypeOopPtr* receiver_type = gvn.type(receiver_node)->isa_oopptr(); 1284 // call_does_dispatch and vtable_index are out-parameters. They might be changed. 1285 // optimize_virtual_call() takes 2 different holder 1286 // arguments for a corner case that doesn't apply here (see 1287 // Parse::do_call()) 1288 target = C->optimize_virtual_call(caller, klass, klass, 1289 target, receiver_type, is_virtual, 1290 call_does_dispatch, vtable_index, // out-parameters 1291 false /* check_access */); 1292 // We lack profiling at this call but type speculation may 1293 // provide us with a type 1294 speculative_receiver_type = (receiver_type != nullptr) ? receiver_type->speculative_type() : nullptr; 1295 } 1296 CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms, 1297 allow_inline, 1298 PROB_ALWAYS, 1299 speculative_receiver_type, 1300 true); 1301 return cg; 1302 } else { 1303 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), 1304 "member_name not constant"); 1305 } 1306 } 1307 break; 1308 1309 case vmIntrinsics::_linkToNative: 1310 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), 1311 "native call"); 1312 break; 1313 1314 default: 1315 fatal("unexpected intrinsic %d: %s", vmIntrinsics::as_int(iid), vmIntrinsics::name_at(iid)); 1316 break; 1317 } 1318 return nullptr; 1319 } 1320 1321 //------------------------PredicatedIntrinsicGenerator------------------------------ 1322 // Internal class which handles all predicated Intrinsic calls. 1323 class PredicatedIntrinsicGenerator : public CallGenerator { 1324 CallGenerator* _intrinsic; 1325 CallGenerator* _cg; 1326 1327 public: 1328 PredicatedIntrinsicGenerator(CallGenerator* intrinsic, 1329 CallGenerator* cg) 1330 : CallGenerator(cg->method()) 1331 { 1332 _intrinsic = intrinsic; 1333 _cg = cg; 1334 } 1335 1336 virtual bool is_virtual() const { return true; } 1337 virtual bool is_inline() const { return true; } 1338 virtual bool is_intrinsic() const { return true; } 1339 1340 virtual JVMState* generate(JVMState* jvms); 1341 }; 1342 1343 1344 CallGenerator* CallGenerator::for_predicated_intrinsic(CallGenerator* intrinsic, 1345 CallGenerator* cg) { 1346 return new PredicatedIntrinsicGenerator(intrinsic, cg); 1347 } 1348 1349 1350 JVMState* PredicatedIntrinsicGenerator::generate(JVMState* jvms) { 1351 // The code we want to generate here is: 1352 // if (receiver == nullptr) 1353 // uncommon_Trap 1354 // if (predicate(0)) 1355 // do_intrinsic(0) 1356 // else 1357 // if (predicate(1)) 1358 // do_intrinsic(1) 1359 // ... 1360 // else 1361 // do_java_comp 1362 1363 GraphKit kit(jvms); 1364 PhaseGVN& gvn = kit.gvn(); 1365 1366 CompileLog* log = kit.C->log(); 1367 if (log != nullptr) { 1368 log->elem("predicated_intrinsic bci='%d' method='%d'", 1369 jvms->bci(), log->identify(method())); 1370 } 1371 1372 if (!method()->is_static()) { 1373 // We need an explicit receiver null_check before checking its type in predicate. 1374 // We share a map with the caller, so his JVMS gets adjusted. 1375 kit.null_check_receiver_before_call(method()); 1376 if (kit.stopped()) { 1377 return kit.transfer_exceptions_into_jvms(); 1378 } 1379 } 1380 1381 int n_predicates = _intrinsic->predicates_count(); 1382 assert(n_predicates > 0, "sanity"); 1383 1384 JVMState** result_jvms = NEW_RESOURCE_ARRAY(JVMState*, (n_predicates+1)); 1385 1386 // Region for normal compilation code if intrinsic failed. 1387 Node* slow_region = new RegionNode(1); 1388 1389 int results = 0; 1390 for (int predicate = 0; (predicate < n_predicates) && !kit.stopped(); predicate++) { 1391 #ifdef ASSERT 1392 JVMState* old_jvms = kit.jvms(); 1393 SafePointNode* old_map = kit.map(); 1394 Node* old_io = old_map->i_o(); 1395 Node* old_mem = old_map->memory(); 1396 Node* old_exc = old_map->next_exception(); 1397 #endif 1398 Node* else_ctrl = _intrinsic->generate_predicate(kit.sync_jvms(), predicate); 1399 #ifdef ASSERT 1400 // Assert(no_new_memory && no_new_io && no_new_exceptions) after generate_predicate. 1401 assert(old_jvms == kit.jvms(), "generate_predicate should not change jvm state"); 1402 SafePointNode* new_map = kit.map(); 1403 assert(old_io == new_map->i_o(), "generate_predicate should not change i_o"); 1404 assert(old_mem == new_map->memory(), "generate_predicate should not change memory"); 1405 assert(old_exc == new_map->next_exception(), "generate_predicate should not add exceptions"); 1406 #endif 1407 if (!kit.stopped()) { 1408 PreserveJVMState pjvms(&kit); 1409 // Generate intrinsic code: 1410 JVMState* new_jvms = _intrinsic->generate(kit.sync_jvms()); 1411 if (new_jvms == nullptr) { 1412 // Intrinsic failed, use normal compilation path for this predicate. 1413 slow_region->add_req(kit.control()); 1414 } else { 1415 kit.add_exception_states_from(new_jvms); 1416 kit.set_jvms(new_jvms); 1417 if (!kit.stopped()) { 1418 result_jvms[results++] = kit.jvms(); 1419 } 1420 } 1421 } 1422 if (else_ctrl == nullptr) { 1423 else_ctrl = kit.C->top(); 1424 } 1425 kit.set_control(else_ctrl); 1426 } 1427 if (!kit.stopped()) { 1428 // Final 'else' after predicates. 1429 slow_region->add_req(kit.control()); 1430 } 1431 if (slow_region->req() > 1) { 1432 PreserveJVMState pjvms(&kit); 1433 // Generate normal compilation code: 1434 kit.set_control(gvn.transform(slow_region)); 1435 JVMState* new_jvms = _cg->generate(kit.sync_jvms()); 1436 if (kit.failing()) 1437 return nullptr; // might happen because of NodeCountInliningCutoff 1438 assert(new_jvms != nullptr, "must be"); 1439 kit.add_exception_states_from(new_jvms); 1440 kit.set_jvms(new_jvms); 1441 if (!kit.stopped()) { 1442 result_jvms[results++] = kit.jvms(); 1443 } 1444 } 1445 1446 if (results == 0) { 1447 // All paths ended in uncommon traps. 1448 (void) kit.stop(); 1449 return kit.transfer_exceptions_into_jvms(); 1450 } 1451 1452 if (results == 1) { // Only one path 1453 kit.set_jvms(result_jvms[0]); 1454 return kit.transfer_exceptions_into_jvms(); 1455 } 1456 1457 // Merge all paths. 1458 kit.C->set_has_split_ifs(true); // Has chance for split-if optimization 1459 RegionNode* region = new RegionNode(results + 1); 1460 Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO); 1461 for (int i = 0; i < results; i++) { 1462 JVMState* jvms = result_jvms[i]; 1463 int path = i + 1; 1464 SafePointNode* map = jvms->map(); 1465 region->init_req(path, map->control()); 1466 iophi->set_req(path, map->i_o()); 1467 if (i == 0) { 1468 kit.set_jvms(jvms); 1469 } else { 1470 kit.merge_memory(map->merged_memory(), region, path); 1471 } 1472 } 1473 kit.set_control(gvn.transform(region)); 1474 kit.set_i_o(gvn.transform(iophi)); 1475 // Transform new memory Phis. 1476 for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) { 1477 Node* phi = mms.memory(); 1478 if (phi->is_Phi() && phi->in(0) == region) { 1479 mms.set_memory(gvn.transform(phi)); 1480 } 1481 } 1482 1483 // Merge debug info. 1484 Node** ins = NEW_RESOURCE_ARRAY(Node*, results); 1485 uint tos = kit.jvms()->stkoff() + kit.sp(); 1486 Node* map = kit.map(); 1487 uint limit = map->req(); 1488 for (uint i = TypeFunc::Parms; i < limit; i++) { 1489 // Skip unused stack slots; fast forward to monoff(); 1490 if (i == tos) { 1491 i = kit.jvms()->monoff(); 1492 if( i >= limit ) break; 1493 } 1494 Node* n = map->in(i); 1495 ins[0] = n; 1496 const Type* t = gvn.type(n); 1497 bool needs_phi = false; 1498 for (int j = 1; j < results; j++) { 1499 JVMState* jvms = result_jvms[j]; 1500 Node* jmap = jvms->map(); 1501 Node* m = nullptr; 1502 if (jmap->req() > i) { 1503 m = jmap->in(i); 1504 if (m != n) { 1505 needs_phi = true; 1506 t = t->meet_speculative(gvn.type(m)); 1507 } 1508 } 1509 ins[j] = m; 1510 } 1511 if (needs_phi) { 1512 Node* phi = PhiNode::make(region, n, t); 1513 for (int j = 1; j < results; j++) { 1514 phi->set_req(j + 1, ins[j]); 1515 } 1516 map->set_req(i, gvn.transform(phi)); 1517 } 1518 } 1519 1520 return kit.transfer_exceptions_into_jvms(); 1521 } 1522 1523 //-------------------------UncommonTrapCallGenerator----------------------------- 1524 // Internal class which handles all out-of-line calls checking receiver type. 1525 class UncommonTrapCallGenerator : public CallGenerator { 1526 Deoptimization::DeoptReason _reason; 1527 Deoptimization::DeoptAction _action; 1528 1529 public: 1530 UncommonTrapCallGenerator(ciMethod* m, 1531 Deoptimization::DeoptReason reason, 1532 Deoptimization::DeoptAction action) 1533 : CallGenerator(m) 1534 { 1535 _reason = reason; 1536 _action = action; 1537 } 1538 1539 virtual bool is_virtual() const { ShouldNotReachHere(); return false; } 1540 virtual bool is_trap() const { return true; } 1541 1542 virtual JVMState* generate(JVMState* jvms); 1543 }; 1544 1545 1546 CallGenerator* 1547 CallGenerator::for_uncommon_trap(ciMethod* m, 1548 Deoptimization::DeoptReason reason, 1549 Deoptimization::DeoptAction action) { 1550 return new UncommonTrapCallGenerator(m, reason, action); 1551 } 1552 1553 1554 JVMState* UncommonTrapCallGenerator::generate(JVMState* jvms) { 1555 GraphKit kit(jvms); 1556 kit.C->print_inlining_update(this); 1557 // Take the trap with arguments pushed on the stack. (Cf. null_check_receiver). 1558 // Callsite signature can be different from actual method being called (i.e _linkTo* sites). 1559 // Use callsite signature always. 1560 ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci()); 1561 int nargs = declared_method->arg_size(); 1562 kit.inc_sp(nargs); 1563 assert(nargs <= kit.sp() && kit.sp() <= jvms->stk_size(), "sane sp w/ args pushed"); 1564 if (_reason == Deoptimization::Reason_class_check && 1565 _action == Deoptimization::Action_maybe_recompile) { 1566 // Temp fix for 6529811 1567 // Don't allow uncommon_trap to override our decision to recompile in the event 1568 // of a class cast failure for a monomorphic call as it will never let us convert 1569 // the call to either bi-morphic or megamorphic and can lead to unc-trap loops 1570 bool keep_exact_action = true; 1571 kit.uncommon_trap(_reason, _action, nullptr, "monomorphic vcall checkcast", false, keep_exact_action); 1572 } else { 1573 kit.uncommon_trap(_reason, _action); 1574 } 1575 return kit.transfer_exceptions_into_jvms(); 1576 } 1577 1578 // (Note: Moved hook_up_call to GraphKit::set_edges_for_java_call.) 1579 1580 // (Node: Merged hook_up_exits into ParseGenerator::generate.)