1 /* 2 * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "ci/bcEscapeAnalyzer.hpp" 27 #include "ci/ciCallSite.hpp" 28 #include "ci/ciObjArray.hpp" 29 #include "ci/ciMemberName.hpp" 30 #include "ci/ciMethodHandle.hpp" 31 #include "classfile/javaClasses.hpp" 32 #include "compiler/compileLog.hpp" 33 #include "opto/addnode.hpp" 34 #include "opto/callGenerator.hpp" 35 #include "opto/callnode.hpp" 36 #include "opto/castnode.hpp" 37 #include "opto/cfgnode.hpp" 38 #include "opto/parse.hpp" 39 #include "opto/rootnode.hpp" 40 #include "opto/runtime.hpp" 41 #include "opto/subnode.hpp" 42 #include "runtime/os.inline.hpp" 43 #include "runtime/sharedRuntime.hpp" 44 #include "utilities/debug.hpp" 45 46 // Utility function. 47 const TypeFunc* CallGenerator::tf() const { 48 return TypeFunc::make(method()); 49 } 50 51 bool CallGenerator::is_inlined_method_handle_intrinsic(JVMState* jvms, ciMethod* m) { 52 return is_inlined_method_handle_intrinsic(jvms->method(), jvms->bci(), m); 53 } 54 55 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* caller, int bci, ciMethod* m) { 56 ciMethod* symbolic_info = caller->get_method_at_bci(bci); 57 return is_inlined_method_handle_intrinsic(symbolic_info, m); 58 } 59 60 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* symbolic_info, ciMethod* m) { 61 return symbolic_info->is_method_handle_intrinsic() && !m->is_method_handle_intrinsic(); 62 } 63 64 //-----------------------------ParseGenerator--------------------------------- 65 // Internal class which handles all direct bytecode traversal. 66 class ParseGenerator : public InlineCallGenerator { 67 private: 68 bool _is_osr; 69 float _expected_uses; 70 71 public: 72 ParseGenerator(ciMethod* method, float expected_uses, bool is_osr = false) 73 : InlineCallGenerator(method) 74 { 75 _is_osr = is_osr; 76 _expected_uses = expected_uses; 77 assert(InlineTree::check_can_parse(method) == nullptr, "parse must be possible"); 78 } 79 80 virtual bool is_parse() const { return true; } 81 virtual JVMState* generate(JVMState* jvms); 82 int is_osr() { return _is_osr; } 83 84 }; 85 86 JVMState* ParseGenerator::generate(JVMState* jvms) { 87 Compile* C = Compile::current(); 88 C->print_inlining_update(this); 89 90 if (is_osr()) { 91 // The JVMS for a OSR has a single argument (see its TypeFunc). 92 assert(jvms->depth() == 1, "no inline OSR"); 93 } 94 95 if (C->failing()) { 96 return nullptr; // bailing out of the compile; do not try to parse 97 } 98 99 Parse parser(jvms, method(), _expected_uses); 100 if (C->failing()) return nullptr; 101 102 // Grab signature for matching/allocation 103 GraphKit& exits = parser.exits(); 104 105 if (C->failing()) { 106 while (exits.pop_exception_state() != nullptr) ; 107 return nullptr; 108 } 109 110 assert(exits.jvms()->same_calls_as(jvms), "sanity"); 111 112 // Simply return the exit state of the parser, 113 // augmented by any exceptional states. 114 return exits.transfer_exceptions_into_jvms(); 115 } 116 117 //---------------------------DirectCallGenerator------------------------------ 118 // Internal class which handles all out-of-line calls w/o receiver type checks. 119 class DirectCallGenerator : public CallGenerator { 120 private: 121 CallStaticJavaNode* _call_node; 122 // Force separate memory and I/O projections for the exceptional 123 // paths to facilitate late inlinig. 124 bool _separate_io_proj; 125 126 protected: 127 void set_call_node(CallStaticJavaNode* call) { _call_node = call; } 128 129 public: 130 DirectCallGenerator(ciMethod* method, bool separate_io_proj) 131 : CallGenerator(method), 132 _separate_io_proj(separate_io_proj) 133 { 134 } 135 virtual JVMState* generate(JVMState* jvms); 136 137 virtual CallNode* call_node() const { return _call_node; } 138 virtual CallGenerator* with_call_node(CallNode* call) { 139 DirectCallGenerator* dcg = new DirectCallGenerator(method(), _separate_io_proj); 140 dcg->set_call_node(call->as_CallStaticJava()); 141 return dcg; 142 } 143 }; 144 145 JVMState* DirectCallGenerator::generate(JVMState* jvms) { 146 GraphKit kit(jvms); 147 kit.C->print_inlining_update(this); 148 bool is_static = method()->is_static(); 149 address target = is_static ? SharedRuntime::get_resolve_static_call_stub() 150 : SharedRuntime::get_resolve_opt_virtual_call_stub(); 151 152 if (kit.C->log() != nullptr) { 153 kit.C->log()->elem("direct_call bci='%d'", jvms->bci()); 154 } 155 156 CallStaticJavaNode* call = new CallStaticJavaNode(kit.C, tf(), target, method()); 157 if (is_inlined_method_handle_intrinsic(jvms, method())) { 158 // To be able to issue a direct call and skip a call to MH.linkTo*/invokeBasic adapter, 159 // additional information about the method being invoked should be attached 160 // to the call site to make resolution logic work 161 // (see SharedRuntime::resolve_static_call_C). 162 call->set_override_symbolic_info(true); 163 } 164 _call_node = call; // Save the call node in case we need it later 165 if (!is_static) { 166 // Make an explicit receiver null_check as part of this call. 167 // Since we share a map with the caller, his JVMS gets adjusted. 168 kit.null_check_receiver_before_call(method()); 169 if (kit.stopped()) { 170 // And dump it back to the caller, decorated with any exceptions: 171 return kit.transfer_exceptions_into_jvms(); 172 } 173 // Mark the call node as virtual, sort of: 174 call->set_optimized_virtual(true); 175 if (method()->is_method_handle_intrinsic() || 176 method()->is_compiled_lambda_form()) { 177 call->set_method_handle_invoke(true); 178 } 179 } 180 kit.set_arguments_for_java_call(call); 181 kit.set_edges_for_java_call(call, false, _separate_io_proj); 182 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj); 183 kit.push_node(method()->return_type()->basic_type(), ret); 184 return kit.transfer_exceptions_into_jvms(); 185 } 186 187 //--------------------------VirtualCallGenerator------------------------------ 188 // Internal class which handles all out-of-line calls checking receiver type. 189 class VirtualCallGenerator : public CallGenerator { 190 private: 191 int _vtable_index; 192 bool _separate_io_proj; 193 CallDynamicJavaNode* _call_node; 194 195 protected: 196 void set_call_node(CallDynamicJavaNode* call) { _call_node = call; } 197 198 public: 199 VirtualCallGenerator(ciMethod* method, int vtable_index, bool separate_io_proj) 200 : CallGenerator(method), _vtable_index(vtable_index), _separate_io_proj(separate_io_proj), _call_node(nullptr) 201 { 202 assert(vtable_index == Method::invalid_vtable_index || 203 vtable_index >= 0, "either invalid or usable"); 204 } 205 virtual bool is_virtual() const { return true; } 206 virtual JVMState* generate(JVMState* jvms); 207 208 virtual CallNode* call_node() const { return _call_node; } 209 int vtable_index() const { return _vtable_index; } 210 211 virtual CallGenerator* with_call_node(CallNode* call) { 212 VirtualCallGenerator* cg = new VirtualCallGenerator(method(), _vtable_index, _separate_io_proj); 213 cg->set_call_node(call->as_CallDynamicJava()); 214 return cg; 215 } 216 }; 217 218 JVMState* VirtualCallGenerator::generate(JVMState* jvms) { 219 GraphKit kit(jvms); 220 Node* receiver = kit.argument(0); 221 222 kit.C->print_inlining_update(this); 223 224 if (kit.C->log() != nullptr) { 225 kit.C->log()->elem("virtual_call bci='%d'", jvms->bci()); 226 } 227 228 // If the receiver is a constant null, do not torture the system 229 // by attempting to call through it. The compile will proceed 230 // correctly, but may bail out in final_graph_reshaping, because 231 // the call instruction will have a seemingly deficient out-count. 232 // (The bailout says something misleading about an "infinite loop".) 233 if (kit.gvn().type(receiver)->higher_equal(TypePtr::NULL_PTR)) { 234 assert(Bytecodes::is_invoke(kit.java_bc()), "%d: %s", kit.java_bc(), Bytecodes::name(kit.java_bc())); 235 ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci()); 236 int arg_size = declared_method->signature()->arg_size_for_bc(kit.java_bc()); 237 kit.inc_sp(arg_size); // restore arguments 238 kit.uncommon_trap(Deoptimization::Reason_null_check, 239 Deoptimization::Action_none, 240 nullptr, "null receiver"); 241 return kit.transfer_exceptions_into_jvms(); 242 } 243 244 // Ideally we would unconditionally do a null check here and let it 245 // be converted to an implicit check based on profile information. 246 // However currently the conversion to implicit null checks in 247 // Block::implicit_null_check() only looks for loads and stores, not calls. 248 ciMethod *caller = kit.method(); 249 ciMethodData *caller_md = (caller == nullptr) ? nullptr : caller->method_data(); 250 if (!UseInlineCaches || !ImplicitNullChecks || !os::zero_page_read_protected() || 251 ((ImplicitNullCheckThreshold > 0) && caller_md && 252 (caller_md->trap_count(Deoptimization::Reason_null_check) 253 >= (uint)ImplicitNullCheckThreshold))) { 254 // Make an explicit receiver null_check as part of this call. 255 // Since we share a map with the caller, his JVMS gets adjusted. 256 receiver = kit.null_check_receiver_before_call(method()); 257 if (kit.stopped()) { 258 // And dump it back to the caller, decorated with any exceptions: 259 return kit.transfer_exceptions_into_jvms(); 260 } 261 } 262 263 assert(!method()->is_static(), "virtual call must not be to static"); 264 assert(!method()->is_final(), "virtual call should not be to final"); 265 assert(!method()->is_private(), "virtual call should not be to private"); 266 assert(_vtable_index == Method::invalid_vtable_index || !UseInlineCaches, 267 "no vtable calls if +UseInlineCaches "); 268 address target = SharedRuntime::get_resolve_virtual_call_stub(); 269 // Normal inline cache used for call 270 CallDynamicJavaNode* call = new CallDynamicJavaNode(tf(), target, method(), _vtable_index); 271 if (is_inlined_method_handle_intrinsic(jvms, method())) { 272 // To be able to issue a direct call (optimized virtual or virtual) 273 // and skip a call to MH.linkTo*/invokeBasic adapter, additional information 274 // about the method being invoked should be attached to the call site to 275 // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C). 276 call->set_override_symbolic_info(true); 277 } 278 _call_node = call; // Save the call node in case we need it later 279 280 kit.set_arguments_for_java_call(call); 281 kit.set_edges_for_java_call(call, false /*must_throw*/, _separate_io_proj); 282 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj); 283 kit.push_node(method()->return_type()->basic_type(), ret); 284 285 // Represent the effect of an implicit receiver null_check 286 // as part of this call. Since we share a map with the caller, 287 // his JVMS gets adjusted. 288 kit.cast_not_null(receiver); 289 return kit.transfer_exceptions_into_jvms(); 290 } 291 292 CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) { 293 if (InlineTree::check_can_parse(m) != nullptr) return nullptr; 294 return new ParseGenerator(m, expected_uses); 295 } 296 297 // As a special case, the JVMS passed to this CallGenerator is 298 // for the method execution already in progress, not just the JVMS 299 // of the caller. Thus, this CallGenerator cannot be mixed with others! 300 CallGenerator* CallGenerator::for_osr(ciMethod* m, int osr_bci) { 301 if (InlineTree::check_can_parse(m) != nullptr) return nullptr; 302 float past_uses = m->interpreter_invocation_count(); 303 float expected_uses = past_uses; 304 return new ParseGenerator(m, expected_uses, true); 305 } 306 307 CallGenerator* CallGenerator::for_direct_call(ciMethod* m, bool separate_io_proj) { 308 assert(!m->is_abstract(), "for_direct_call mismatch"); 309 return new DirectCallGenerator(m, separate_io_proj); 310 } 311 312 CallGenerator* CallGenerator::for_virtual_call(ciMethod* m, int vtable_index) { 313 assert(!m->is_static(), "for_virtual_call mismatch"); 314 assert(!m->is_method_handle_intrinsic(), "should be a direct call"); 315 return new VirtualCallGenerator(m, vtable_index, false /*separate_io_projs*/); 316 } 317 318 // Allow inlining decisions to be delayed 319 class LateInlineCallGenerator : public DirectCallGenerator { 320 private: 321 jlong _unique_id; // unique id for log compilation 322 bool _is_pure_call; // a hint that the call doesn't have important side effects to care about 323 324 protected: 325 CallGenerator* _inline_cg; 326 virtual bool do_late_inline_check(Compile* C, JVMState* jvms) { return true; } 327 virtual CallGenerator* inline_cg() const { return _inline_cg; } 328 virtual bool is_pure_call() const { return _is_pure_call; } 329 330 public: 331 LateInlineCallGenerator(ciMethod* method, CallGenerator* inline_cg, bool is_pure_call = false) : 332 DirectCallGenerator(method, true), _unique_id(0), _is_pure_call(is_pure_call), _inline_cg(inline_cg) {} 333 334 virtual bool is_late_inline() const { return true; } 335 336 // Convert the CallStaticJava into an inline 337 virtual void do_late_inline(); 338 339 virtual JVMState* generate(JVMState* jvms) { 340 Compile *C = Compile::current(); 341 342 C->log_inline_id(this); 343 344 // Record that this call site should be revisited once the main 345 // parse is finished. 346 if (!is_mh_late_inline()) { 347 C->add_late_inline(this); 348 } 349 350 // Emit the CallStaticJava and request separate projections so 351 // that the late inlining logic can distinguish between fall 352 // through and exceptional uses of the memory and io projections 353 // as is done for allocations and macro expansion. 354 return DirectCallGenerator::generate(jvms); 355 } 356 357 virtual void print_inlining_late(InliningResult result, const char* msg) { 358 CallNode* call = call_node(); 359 Compile* C = Compile::current(); 360 C->print_inlining_assert_ready(); 361 C->print_inlining(method(), call->jvms()->depth()-1, call->jvms()->bci(), result, msg); 362 C->print_inlining_move_to(this); 363 C->print_inlining_update_delayed(this); 364 } 365 366 virtual void set_unique_id(jlong id) { 367 _unique_id = id; 368 } 369 370 virtual jlong unique_id() const { 371 return _unique_id; 372 } 373 374 virtual CallGenerator* with_call_node(CallNode* call) { 375 LateInlineCallGenerator* cg = new LateInlineCallGenerator(method(), _inline_cg, _is_pure_call); 376 cg->set_call_node(call->as_CallStaticJava()); 377 return cg; 378 } 379 }; 380 381 CallGenerator* CallGenerator::for_late_inline(ciMethod* method, CallGenerator* inline_cg) { 382 return new LateInlineCallGenerator(method, inline_cg); 383 } 384 385 class LateInlineMHCallGenerator : public LateInlineCallGenerator { 386 ciMethod* _caller; 387 bool _input_not_const; 388 389 virtual bool do_late_inline_check(Compile* C, JVMState* jvms); 390 391 public: 392 LateInlineMHCallGenerator(ciMethod* caller, ciMethod* callee, bool input_not_const) : 393 LateInlineCallGenerator(callee, nullptr), _caller(caller), _input_not_const(input_not_const) {} 394 395 virtual bool is_mh_late_inline() const { return true; } 396 397 // Convert the CallStaticJava into an inline 398 virtual void do_late_inline(); 399 400 virtual JVMState* generate(JVMState* jvms) { 401 JVMState* new_jvms = LateInlineCallGenerator::generate(jvms); 402 403 Compile* C = Compile::current(); 404 if (_input_not_const) { 405 // inlining won't be possible so no need to enqueue right now. 406 call_node()->set_generator(this); 407 } else { 408 C->add_late_inline(this); 409 } 410 return new_jvms; 411 } 412 413 virtual CallGenerator* with_call_node(CallNode* call) { 414 LateInlineMHCallGenerator* cg = new LateInlineMHCallGenerator(_caller, method(), _input_not_const); 415 cg->set_call_node(call->as_CallStaticJava()); 416 return cg; 417 } 418 }; 419 420 bool LateInlineMHCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) { 421 // When inlining a virtual call, the null check at the call and the call itself can throw. These 2 paths have different 422 // expression stacks which causes late inlining to break. The MH invoker is not expected to be called from a method with 423 // exception handlers. When there is no exception handler, GraphKit::builtin_throw() pops the stack which solves the issue 424 // of late inlining with exceptions. 425 assert(!jvms->method()->has_exception_handlers() || 426 (method()->intrinsic_id() != vmIntrinsics::_linkToVirtual && 427 method()->intrinsic_id() != vmIntrinsics::_linkToInterface), "no exception handler expected"); 428 // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call. 429 bool allow_inline = C->inlining_incrementally(); 430 bool input_not_const = true; 431 CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), allow_inline, input_not_const); 432 assert(!input_not_const, "sanity"); // shouldn't have been scheduled for inlining in the first place 433 434 if (cg != nullptr) { 435 assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline || StressIncrementalInlining, "we're doing late inlining"); 436 _inline_cg = cg; 437 C->dec_number_of_mh_late_inlines(); 438 return true; 439 } else { 440 // Method handle call which has a constant appendix argument should be either inlined or replaced with a direct call 441 // unless there's a signature mismatch between caller and callee. If the failure occurs, there's not much to be improved later, 442 // so don't reinstall the generator to avoid pushing the generator between IGVN and incremental inlining indefinitely. 443 return false; 444 } 445 } 446 447 CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const) { 448 assert(IncrementalInlineMH, "required"); 449 Compile::current()->inc_number_of_mh_late_inlines(); 450 CallGenerator* cg = new LateInlineMHCallGenerator(caller, callee, input_not_const); 451 return cg; 452 } 453 454 // Allow inlining decisions to be delayed 455 class LateInlineVirtualCallGenerator : public VirtualCallGenerator { 456 private: 457 jlong _unique_id; // unique id for log compilation 458 CallGenerator* _inline_cg; 459 ciMethod* _callee; 460 bool _is_pure_call; 461 float _prof_factor; 462 463 protected: 464 virtual bool do_late_inline_check(Compile* C, JVMState* jvms); 465 virtual CallGenerator* inline_cg() const { return _inline_cg; } 466 virtual bool is_pure_call() const { return _is_pure_call; } 467 468 public: 469 LateInlineVirtualCallGenerator(ciMethod* method, int vtable_index, float prof_factor) 470 : VirtualCallGenerator(method, vtable_index, true /*separate_io_projs*/), 471 _unique_id(0), _inline_cg(nullptr), _callee(nullptr), _is_pure_call(false), _prof_factor(prof_factor) { 472 assert(IncrementalInlineVirtual, "required"); 473 } 474 475 virtual bool is_late_inline() const { return true; } 476 477 virtual bool is_virtual_late_inline() const { return true; } 478 479 // Convert the CallDynamicJava into an inline 480 virtual void do_late_inline(); 481 482 virtual void set_callee_method(ciMethod* m) { 483 assert(_callee == nullptr, "repeated inlining attempt"); 484 _callee = m; 485 } 486 487 virtual JVMState* generate(JVMState* jvms) { 488 // Emit the CallDynamicJava and request separate projections so 489 // that the late inlining logic can distinguish between fall 490 // through and exceptional uses of the memory and io projections 491 // as is done for allocations and macro expansion. 492 JVMState* new_jvms = VirtualCallGenerator::generate(jvms); 493 if (call_node() != nullptr) { 494 call_node()->set_generator(this); 495 } 496 return new_jvms; 497 } 498 499 virtual void print_inlining_late(InliningResult result, const char* msg) { 500 CallNode* call = call_node(); 501 Compile* C = Compile::current(); 502 C->print_inlining_assert_ready(); 503 C->print_inlining(method(), call->jvms()->depth()-1, call->jvms()->bci(), result, msg); 504 C->print_inlining_move_to(this); 505 C->print_inlining_update_delayed(this); 506 } 507 508 virtual void set_unique_id(jlong id) { 509 _unique_id = id; 510 } 511 512 virtual jlong unique_id() const { 513 return _unique_id; 514 } 515 516 virtual CallGenerator* with_call_node(CallNode* call) { 517 LateInlineVirtualCallGenerator* cg = new LateInlineVirtualCallGenerator(method(), vtable_index(), _prof_factor); 518 cg->set_call_node(call->as_CallDynamicJava()); 519 return cg; 520 } 521 }; 522 523 bool LateInlineVirtualCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) { 524 // Method handle linker case is handled in CallDynamicJavaNode::Ideal(). 525 // Unless inlining is performed, _override_symbolic_info bit will be set in DirectCallGenerator::generate(). 526 527 // Implicit receiver null checks introduce problems when exception states are combined. 528 Node* receiver = jvms->map()->argument(jvms, 0); 529 const Type* recv_type = C->initial_gvn()->type(receiver); 530 if (recv_type->maybe_null()) { 531 if (C->print_inlining() || C->print_intrinsics()) { 532 C->print_inlining(method(), jvms->depth()-1, call_node()->jvms()->bci(), InliningResult::FAILURE, 533 "late call devirtualization failed (receiver may be null)"); 534 } 535 return false; 536 } 537 // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call. 538 bool allow_inline = C->inlining_incrementally(); 539 if (!allow_inline && _callee->holder()->is_interface()) { 540 // Don't convert the interface call to a direct call guarded by an interface subtype check. 541 if (C->print_inlining() || C->print_intrinsics()) { 542 C->print_inlining(method(), jvms->depth()-1, call_node()->jvms()->bci(), InliningResult::FAILURE, 543 "late call devirtualization failed (interface call)"); 544 } 545 return false; 546 } 547 CallGenerator* cg = C->call_generator(_callee, 548 vtable_index(), 549 false /*call_does_dispatch*/, 550 jvms, 551 allow_inline, 552 _prof_factor, 553 nullptr /*speculative_receiver_type*/, 554 true /*allow_intrinsics*/); 555 556 if (cg != nullptr) { 557 assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline || StressIncrementalInlining, "we're doing late inlining"); 558 _inline_cg = cg; 559 return true; 560 } else { 561 // Virtual call which provably doesn't dispatch should be either inlined or replaced with a direct call. 562 assert(false, "no progress"); 563 return false; 564 } 565 } 566 567 CallGenerator* CallGenerator::for_late_inline_virtual(ciMethod* m, int vtable_index, float prof_factor) { 568 assert(IncrementalInlineVirtual, "required"); 569 assert(!m->is_static(), "for_virtual_call mismatch"); 570 assert(!m->is_method_handle_intrinsic(), "should be a direct call"); 571 return new LateInlineVirtualCallGenerator(m, vtable_index, prof_factor); 572 } 573 574 void LateInlineCallGenerator::do_late_inline() { 575 CallGenerator::do_late_inline_helper(); 576 } 577 578 void LateInlineMHCallGenerator::do_late_inline() { 579 CallGenerator::do_late_inline_helper(); 580 } 581 582 void LateInlineVirtualCallGenerator::do_late_inline() { 583 assert(_callee != nullptr, "required"); // set up in CallDynamicJavaNode::Ideal 584 CallGenerator::do_late_inline_helper(); 585 } 586 587 void CallGenerator::do_late_inline_helper() { 588 assert(is_late_inline(), "only late inline allowed"); 589 590 // Can't inline it 591 CallNode* call = call_node(); 592 if (call == nullptr || call->outcnt() == 0 || 593 call->in(0) == nullptr || call->in(0)->is_top()) { 594 return; 595 } 596 597 const TypeTuple *r = call->tf()->domain(); 598 for (int i1 = 0; i1 < method()->arg_size(); i1++) { 599 if (call->in(TypeFunc::Parms + i1)->is_top() && r->field_at(TypeFunc::Parms + i1) != Type::HALF) { 600 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing"); 601 return; 602 } 603 } 604 605 if (call->in(TypeFunc::Memory)->is_top()) { 606 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing"); 607 return; 608 } 609 if (call->in(TypeFunc::Memory)->is_MergeMem()) { 610 MergeMemNode* merge_mem = call->in(TypeFunc::Memory)->as_MergeMem(); 611 if (merge_mem->base_memory() == merge_mem->empty_memory()) { 612 return; // dead path 613 } 614 } 615 616 // check for unreachable loop 617 CallProjections callprojs; 618 call->extract_projections(&callprojs, true); 619 if ((callprojs.fallthrough_catchproj == call->in(0)) || 620 (callprojs.catchall_catchproj == call->in(0)) || 621 (callprojs.fallthrough_memproj == call->in(TypeFunc::Memory)) || 622 (callprojs.catchall_memproj == call->in(TypeFunc::Memory)) || 623 (callprojs.fallthrough_ioproj == call->in(TypeFunc::I_O)) || 624 (callprojs.catchall_ioproj == call->in(TypeFunc::I_O)) || 625 (callprojs.resproj != nullptr && call->find_edge(callprojs.resproj) != -1) || 626 (callprojs.exobj != nullptr && call->find_edge(callprojs.exobj) != -1)) { 627 return; 628 } 629 630 Compile* C = Compile::current(); 631 // Remove inlined methods from Compiler's lists. 632 if (call->is_macro()) { 633 C->remove_macro_node(call); 634 } 635 636 // The call is marked as pure (no important side effects), but result isn't used. 637 // It's safe to remove the call. 638 bool result_not_used = (callprojs.resproj == nullptr || callprojs.resproj->outcnt() == 0); 639 640 if (is_pure_call() && result_not_used) { 641 GraphKit kit(call->jvms()); 642 kit.replace_call(call, C->top(), true); 643 } else { 644 // Make a clone of the JVMState that appropriate to use for driving a parse 645 JVMState* old_jvms = call->jvms(); 646 JVMState* jvms = old_jvms->clone_shallow(C); 647 648 // Clear the allocation state. We assume all inputs are materialized. 649 jvms->alloc_state().clear(); 650 651 uint size = call->req(); 652 SafePointNode* map = new SafePointNode(size, jvms); 653 for (uint i1 = 0; i1 < size; i1++) { 654 map->init_req(i1, call->in(i1)); 655 } 656 657 // Make sure the state is a MergeMem for parsing. 658 if (!map->in(TypeFunc::Memory)->is_MergeMem()) { 659 Node* mem = MergeMemNode::make(map->in(TypeFunc::Memory)); 660 C->initial_gvn()->set_type_bottom(mem); 661 map->set_req(TypeFunc::Memory, mem); 662 } 663 664 uint nargs = method()->arg_size(); 665 // blow away old call arguments 666 Node* top = C->top(); 667 for (uint i1 = 0; i1 < nargs; i1++) { 668 map->set_req(TypeFunc::Parms + i1, top); 669 } 670 jvms->set_map(map); 671 672 // Make enough space in the expression stack to transfer 673 // the incoming arguments and return value. 674 map->ensure_stack(jvms, jvms->method()->max_stack()); 675 for (uint i1 = 0; i1 < nargs; i1++) { 676 map->set_argument(jvms, i1, call->in(TypeFunc::Parms + i1)); 677 } 678 679 C->print_inlining_assert_ready(); 680 681 C->print_inlining_move_to(this); 682 683 C->log_late_inline(this); 684 685 // JVMState is ready, so time to perform some checks and prepare for inlining attempt. 686 if (!do_late_inline_check(C, jvms)) { 687 map->disconnect_inputs(C); 688 C->print_inlining_update_delayed(this); 689 return; 690 } 691 692 // Setup default node notes to be picked up by the inlining 693 Node_Notes* old_nn = C->node_notes_at(call->_idx); 694 if (old_nn != nullptr) { 695 Node_Notes* entry_nn = old_nn->clone(C); 696 entry_nn->set_jvms(jvms); 697 C->set_default_node_notes(entry_nn); 698 } 699 700 // Now perform the inlining using the synthesized JVMState 701 JVMState* new_jvms = inline_cg()->generate(jvms); 702 if (new_jvms == nullptr) return; // no change 703 if (C->failing()) return; 704 705 // Capture any exceptional control flow 706 GraphKit kit(new_jvms); 707 708 // Find the result object 709 Node* result = C->top(); 710 int result_size = method()->return_type()->size(); 711 if (result_size != 0 && !kit.stopped()) { 712 result = (result_size == 1) ? kit.pop() : kit.pop_pair(); 713 } 714 715 if (call->is_CallStaticJava() && call->as_CallStaticJava()->is_boxing_method()) { 716 result = kit.must_be_not_null(result, false); 717 } 718 719 if (inline_cg()->is_inline()) { 720 C->set_has_loops(C->has_loops() || inline_cg()->method()->has_loops()); 721 C->env()->notice_inlined_method(inline_cg()->method()); 722 } 723 C->set_inlining_progress(true); 724 C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup 725 kit.replace_call(call, result, true); 726 } 727 } 728 729 class LateInlineStringCallGenerator : public LateInlineCallGenerator { 730 731 public: 732 LateInlineStringCallGenerator(ciMethod* method, CallGenerator* inline_cg) : 733 LateInlineCallGenerator(method, inline_cg) {} 734 735 virtual JVMState* generate(JVMState* jvms) { 736 Compile *C = Compile::current(); 737 738 C->log_inline_id(this); 739 740 C->add_string_late_inline(this); 741 742 JVMState* new_jvms = DirectCallGenerator::generate(jvms); 743 return new_jvms; 744 } 745 746 virtual bool is_string_late_inline() const { return true; } 747 748 virtual CallGenerator* with_call_node(CallNode* call) { 749 LateInlineStringCallGenerator* cg = new LateInlineStringCallGenerator(method(), _inline_cg); 750 cg->set_call_node(call->as_CallStaticJava()); 751 return cg; 752 } 753 }; 754 755 CallGenerator* CallGenerator::for_string_late_inline(ciMethod* method, CallGenerator* inline_cg) { 756 return new LateInlineStringCallGenerator(method, inline_cg); 757 } 758 759 class LateInlineBoxingCallGenerator : public LateInlineCallGenerator { 760 761 public: 762 LateInlineBoxingCallGenerator(ciMethod* method, CallGenerator* inline_cg) : 763 LateInlineCallGenerator(method, inline_cg, /*is_pure=*/true) {} 764 765 virtual JVMState* generate(JVMState* jvms) { 766 Compile *C = Compile::current(); 767 768 C->log_inline_id(this); 769 770 C->add_boxing_late_inline(this); 771 772 JVMState* new_jvms = DirectCallGenerator::generate(jvms); 773 return new_jvms; 774 } 775 776 virtual CallGenerator* with_call_node(CallNode* call) { 777 LateInlineBoxingCallGenerator* cg = new LateInlineBoxingCallGenerator(method(), _inline_cg); 778 cg->set_call_node(call->as_CallStaticJava()); 779 return cg; 780 } 781 }; 782 783 CallGenerator* CallGenerator::for_boxing_late_inline(ciMethod* method, CallGenerator* inline_cg) { 784 return new LateInlineBoxingCallGenerator(method, inline_cg); 785 } 786 787 class LateInlineVectorReboxingCallGenerator : public LateInlineCallGenerator { 788 789 public: 790 LateInlineVectorReboxingCallGenerator(ciMethod* method, CallGenerator* inline_cg) : 791 LateInlineCallGenerator(method, inline_cg, /*is_pure=*/true) {} 792 793 virtual JVMState* generate(JVMState* jvms) { 794 Compile *C = Compile::current(); 795 796 C->log_inline_id(this); 797 798 C->add_vector_reboxing_late_inline(this); 799 800 JVMState* new_jvms = DirectCallGenerator::generate(jvms); 801 return new_jvms; 802 } 803 804 virtual CallGenerator* with_call_node(CallNode* call) { 805 LateInlineVectorReboxingCallGenerator* cg = new LateInlineVectorReboxingCallGenerator(method(), _inline_cg); 806 cg->set_call_node(call->as_CallStaticJava()); 807 return cg; 808 } 809 }; 810 811 // static CallGenerator* for_vector_reboxing_late_inline(ciMethod* m, CallGenerator* inline_cg); 812 CallGenerator* CallGenerator::for_vector_reboxing_late_inline(ciMethod* method, CallGenerator* inline_cg) { 813 return new LateInlineVectorReboxingCallGenerator(method, inline_cg); 814 } 815 816 //------------------------PredictedCallGenerator------------------------------ 817 // Internal class which handles all out-of-line calls checking receiver type. 818 class PredictedCallGenerator : public InlineCallGenerator { 819 ciKlass* _predicted_receiver; 820 CallGenerator* _if_missed; 821 CallGenerator* _if_hit; 822 float _hit_prob; 823 bool _exact_check; 824 825 public: 826 PredictedCallGenerator(ciKlass* predicted_receiver, 827 CallGenerator* if_missed, 828 CallGenerator* if_hit, bool exact_check, 829 float hit_prob) 830 : InlineCallGenerator(if_missed->method()) 831 { 832 // The call profile data may predict the hit_prob as extreme as 0 or 1. 833 // Remove the extremes values from the range. 834 if (hit_prob > PROB_MAX) hit_prob = PROB_MAX; 835 if (hit_prob < PROB_MIN) hit_prob = PROB_MIN; 836 837 _predicted_receiver = predicted_receiver; 838 _if_missed = if_missed; 839 _if_hit = if_hit; 840 _hit_prob = hit_prob; 841 _exact_check = exact_check; 842 } 843 844 virtual bool is_virtual() const { return true; } 845 virtual bool is_inline() const { return _if_hit->is_inline(); } 846 virtual bool is_deferred() const { return _if_hit->is_deferred(); } 847 848 virtual JVMState* generate(JVMState* jvms); 849 }; 850 851 852 CallGenerator* CallGenerator::for_predicted_call(ciKlass* predicted_receiver, 853 CallGenerator* if_missed, 854 CallGenerator* if_hit, 855 float hit_prob) { 856 return new PredictedCallGenerator(predicted_receiver, if_missed, if_hit, 857 /*exact_check=*/true, hit_prob); 858 } 859 860 CallGenerator* CallGenerator::for_guarded_call(ciKlass* guarded_receiver, 861 CallGenerator* if_missed, 862 CallGenerator* if_hit) { 863 return new PredictedCallGenerator(guarded_receiver, if_missed, if_hit, 864 /*exact_check=*/false, PROB_ALWAYS); 865 } 866 867 JVMState* PredictedCallGenerator::generate(JVMState* jvms) { 868 GraphKit kit(jvms); 869 kit.C->print_inlining_update(this); 870 PhaseGVN& gvn = kit.gvn(); 871 // We need an explicit receiver null_check before checking its type. 872 // We share a map with the caller, so his JVMS gets adjusted. 873 Node* receiver = kit.argument(0); 874 CompileLog* log = kit.C->log(); 875 if (log != nullptr) { 876 log->elem("predicted_call bci='%d' exact='%d' klass='%d'", 877 jvms->bci(), (_exact_check ? 1 : 0), log->identify(_predicted_receiver)); 878 } 879 880 receiver = kit.null_check_receiver_before_call(method()); 881 if (kit.stopped()) { 882 return kit.transfer_exceptions_into_jvms(); 883 } 884 885 // Make a copy of the replaced nodes in case we need to restore them 886 ReplacedNodes replaced_nodes = kit.map()->replaced_nodes(); 887 replaced_nodes.clone(); 888 889 Node* casted_receiver = receiver; // will get updated in place... 890 Node* slow_ctl = nullptr; 891 if (_exact_check) { 892 slow_ctl = kit.type_check_receiver(receiver, _predicted_receiver, _hit_prob, 893 &casted_receiver); 894 } else { 895 slow_ctl = kit.subtype_check_receiver(receiver, _predicted_receiver, 896 &casted_receiver); 897 } 898 899 SafePointNode* slow_map = nullptr; 900 JVMState* slow_jvms = nullptr; 901 { PreserveJVMState pjvms(&kit); 902 kit.set_control(slow_ctl); 903 if (!kit.stopped()) { 904 slow_jvms = _if_missed->generate(kit.sync_jvms()); 905 if (kit.failing()) 906 return nullptr; // might happen because of NodeCountInliningCutoff 907 assert(slow_jvms != nullptr, "must be"); 908 kit.add_exception_states_from(slow_jvms); 909 kit.set_map(slow_jvms->map()); 910 if (!kit.stopped()) 911 slow_map = kit.stop(); 912 } 913 } 914 915 if (kit.stopped()) { 916 // Instance does not match the predicted type. 917 kit.set_jvms(slow_jvms); 918 return kit.transfer_exceptions_into_jvms(); 919 } 920 921 // Fall through if the instance matches the desired type. 922 kit.replace_in_map(receiver, casted_receiver); 923 // Make the hot call: 924 JVMState* new_jvms = _if_hit->generate(kit.sync_jvms()); 925 if (new_jvms == nullptr) { 926 // Inline failed, so make a direct call. 927 assert(_if_hit->is_inline(), "must have been a failed inline"); 928 CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method()); 929 new_jvms = cg->generate(kit.sync_jvms()); 930 } 931 kit.add_exception_states_from(new_jvms); 932 kit.set_jvms(new_jvms); 933 934 // Need to merge slow and fast? 935 if (slow_map == nullptr) { 936 // The fast path is the only path remaining. 937 return kit.transfer_exceptions_into_jvms(); 938 } 939 940 if (kit.stopped()) { 941 // Inlined method threw an exception, so it's just the slow path after all. 942 kit.set_jvms(slow_jvms); 943 return kit.transfer_exceptions_into_jvms(); 944 } 945 946 // There are 2 branches and the replaced nodes are only valid on 947 // one: restore the replaced nodes to what they were before the 948 // branch. 949 kit.map()->set_replaced_nodes(replaced_nodes); 950 951 PEAState& slow_as = slow_jvms->alloc_state(); 952 PEAState& as = new_jvms->alloc_state(); 953 AllocationStateMerger as_merger(as); 954 // Finish the diamond. 955 kit.C->set_has_split_ifs(true); // Has chance for split-if optimization 956 RegionNode* region = new RegionNode(3); 957 region->init_req(1, kit.control()); 958 region->init_req(2, slow_map->control()); 959 kit.set_control(gvn.transform(region)); 960 Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO); 961 iophi->set_req(2, slow_map->i_o()); 962 kit.set_i_o(gvn.transform(iophi)); 963 // Merge memory 964 kit.merge_memory(slow_map->merged_memory(), region, 2); 965 // Transform new memory Phis. 966 for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) { 967 Node* phi = mms.memory(); 968 if (phi->is_Phi() && phi->in(0) == region) { 969 mms.set_memory(gvn.transform(phi)); 970 } 971 } 972 uint tos = kit.jvms()->stkoff() + kit.sp(); 973 uint limit = slow_map->req(); 974 for (uint i = TypeFunc::Parms; i < limit; i++) { 975 // Skip unused stack slots; fast forward to monoff(); 976 if (i == tos) { 977 i = kit.jvms()->monoff(); 978 if( i >= limit ) break; 979 } 980 Node* m = kit.map()->in(i); 981 Node* n = slow_map->in(i); 982 if (m != n) { 983 const Type* t = gvn.type(m)->meet_speculative(gvn.type(n)); 984 Node* phi = PhiNode::make(region, m, t); 985 phi->set_req(2, n); 986 kit.map()->set_req(i, gvn.transform(phi)); 987 if (DoPartialEscapeAnalysis) { 988 as_merger.merge_at_phi_creation(kit.PEA(), slow_as, phi->as_Phi(), m, n); 989 } 990 } 991 } 992 993 if (DoPartialEscapeAnalysis) { 994 as_merger.merge(slow_as, &kit, region, 2); 995 } 996 return kit.transfer_exceptions_into_jvms(); 997 } 998 999 1000 CallGenerator* CallGenerator::for_method_handle_call(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline) { 1001 assert(callee->is_method_handle_intrinsic(), "for_method_handle_call mismatch"); 1002 bool input_not_const; 1003 CallGenerator* cg = CallGenerator::for_method_handle_inline(jvms, caller, callee, allow_inline, input_not_const); 1004 Compile* C = Compile::current(); 1005 bool should_delay = C->should_delay_inlining(); 1006 if (cg != nullptr) { 1007 if (should_delay) { 1008 return CallGenerator::for_late_inline(callee, cg); 1009 } else { 1010 return cg; 1011 } 1012 } 1013 int bci = jvms->bci(); 1014 ciCallProfile profile = caller->call_profile_at_bci(bci); 1015 int call_site_count = caller->scale_count(profile.count()); 1016 1017 if (IncrementalInlineMH && call_site_count > 0 && 1018 (should_delay || input_not_const || !C->inlining_incrementally() || C->over_inlining_cutoff())) { 1019 return CallGenerator::for_mh_late_inline(caller, callee, input_not_const); 1020 } else { 1021 // Out-of-line call. 1022 return CallGenerator::for_direct_call(callee); 1023 } 1024 } 1025 1026 CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline, bool& input_not_const) { 1027 GraphKit kit(jvms); 1028 PhaseGVN& gvn = kit.gvn(); 1029 Compile* C = kit.C; 1030 vmIntrinsics::ID iid = callee->intrinsic_id(); 1031 input_not_const = true; 1032 if (StressMethodHandleLinkerInlining) { 1033 allow_inline = false; 1034 } 1035 switch (iid) { 1036 case vmIntrinsics::_invokeBasic: 1037 { 1038 // Get MethodHandle receiver: 1039 Node* receiver = kit.argument(0); 1040 if (receiver->Opcode() == Op_ConP) { 1041 input_not_const = false; 1042 const TypeOopPtr* recv_toop = receiver->bottom_type()->isa_oopptr(); 1043 if (recv_toop != nullptr) { 1044 ciMethod* target = recv_toop->const_oop()->as_method_handle()->get_vmtarget(); 1045 const int vtable_index = Method::invalid_vtable_index; 1046 1047 if (!ciMethod::is_consistent_info(callee, target)) { 1048 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), 1049 "signatures mismatch"); 1050 return nullptr; 1051 } 1052 1053 CallGenerator *cg = C->call_generator(target, vtable_index, 1054 false /* call_does_dispatch */, 1055 jvms, 1056 allow_inline, 1057 PROB_ALWAYS); 1058 return cg; 1059 } else { 1060 assert(receiver->bottom_type() == TypePtr::NULL_PTR, "not a null: %s", 1061 Type::str(receiver->bottom_type())); 1062 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), 1063 "receiver is always null"); 1064 } 1065 } else { 1066 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), 1067 "receiver not constant"); 1068 } 1069 } 1070 break; 1071 1072 case vmIntrinsics::_linkToVirtual: 1073 case vmIntrinsics::_linkToStatic: 1074 case vmIntrinsics::_linkToSpecial: 1075 case vmIntrinsics::_linkToInterface: 1076 { 1077 // Get MemberName argument: 1078 Node* member_name = kit.argument(callee->arg_size() - 1); 1079 if (member_name->Opcode() == Op_ConP) { 1080 input_not_const = false; 1081 const TypeOopPtr* oop_ptr = member_name->bottom_type()->is_oopptr(); 1082 ciMethod* target = oop_ptr->const_oop()->as_member_name()->get_vmtarget(); 1083 1084 if (!ciMethod::is_consistent_info(callee, target)) { 1085 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), 1086 "signatures mismatch"); 1087 return nullptr; 1088 } 1089 1090 // In lambda forms we erase signature types to avoid resolving issues 1091 // involving class loaders. When we optimize a method handle invoke 1092 // to a direct call we must cast the receiver and arguments to its 1093 // actual types. 1094 ciSignature* signature = target->signature(); 1095 const int receiver_skip = target->is_static() ? 0 : 1; 1096 // Cast receiver to its type. 1097 if (!target->is_static()) { 1098 Node* arg = kit.argument(0); 1099 const TypeOopPtr* arg_type = arg->bottom_type()->isa_oopptr(); 1100 const Type* sig_type = TypeOopPtr::make_from_klass(signature->accessing_klass()); 1101 if (arg_type != nullptr && !arg_type->higher_equal(sig_type)) { 1102 const Type* recv_type = arg_type->filter_speculative(sig_type); // keep speculative part 1103 Node* cast_obj = gvn.transform(new CheckCastPPNode(kit.control(), arg, recv_type)); 1104 kit.set_argument(0, cast_obj); 1105 } 1106 } 1107 // Cast reference arguments to its type. 1108 for (int i = 0, j = 0; i < signature->count(); i++) { 1109 ciType* t = signature->type_at(i); 1110 if (t->is_klass()) { 1111 Node* arg = kit.argument(receiver_skip + j); 1112 const TypeOopPtr* arg_type = arg->bottom_type()->isa_oopptr(); 1113 const Type* sig_type = TypeOopPtr::make_from_klass(t->as_klass()); 1114 if (arg_type != nullptr && !arg_type->higher_equal(sig_type)) { 1115 const Type* narrowed_arg_type = arg_type->filter_speculative(sig_type); // keep speculative part 1116 Node* cast_obj = gvn.transform(new CheckCastPPNode(kit.control(), arg, narrowed_arg_type)); 1117 kit.set_argument(receiver_skip + j, cast_obj); 1118 } 1119 } 1120 j += t->size(); // long and double take two slots 1121 } 1122 1123 // Try to get the most accurate receiver type 1124 const bool is_virtual = (iid == vmIntrinsics::_linkToVirtual); 1125 const bool is_virtual_or_interface = (is_virtual || iid == vmIntrinsics::_linkToInterface); 1126 int vtable_index = Method::invalid_vtable_index; 1127 bool call_does_dispatch = false; 1128 1129 ciKlass* speculative_receiver_type = nullptr; 1130 if (is_virtual_or_interface) { 1131 ciInstanceKlass* klass = target->holder(); 1132 Node* receiver_node = kit.argument(0); 1133 const TypeOopPtr* receiver_type = gvn.type(receiver_node)->isa_oopptr(); 1134 // call_does_dispatch and vtable_index are out-parameters. They might be changed. 1135 // optimize_virtual_call() takes 2 different holder 1136 // arguments for a corner case that doesn't apply here (see 1137 // Parse::do_call()) 1138 target = C->optimize_virtual_call(caller, klass, klass, 1139 target, receiver_type, is_virtual, 1140 call_does_dispatch, vtable_index, // out-parameters 1141 false /* check_access */); 1142 // We lack profiling at this call but type speculation may 1143 // provide us with a type 1144 speculative_receiver_type = (receiver_type != nullptr) ? receiver_type->speculative_type() : nullptr; 1145 } 1146 CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms, 1147 allow_inline, 1148 PROB_ALWAYS, 1149 speculative_receiver_type); 1150 return cg; 1151 } else { 1152 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), 1153 "member_name not constant"); 1154 } 1155 } 1156 break; 1157 1158 case vmIntrinsics::_linkToNative: 1159 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(), 1160 "native call"); 1161 break; 1162 1163 default: 1164 fatal("unexpected intrinsic %d: %s", vmIntrinsics::as_int(iid), vmIntrinsics::name_at(iid)); 1165 break; 1166 } 1167 return nullptr; 1168 } 1169 1170 //------------------------PredicatedIntrinsicGenerator------------------------------ 1171 // Internal class which handles all predicated Intrinsic calls. 1172 class PredicatedIntrinsicGenerator : public InlineCallGenerator { 1173 CallGenerator* _intrinsic; 1174 CallGenerator* _cg; 1175 1176 public: 1177 PredicatedIntrinsicGenerator(CallGenerator* intrinsic, 1178 CallGenerator* cg) 1179 : InlineCallGenerator(cg->method()) 1180 { 1181 _intrinsic = intrinsic; 1182 _cg = cg; 1183 } 1184 1185 virtual bool is_virtual() const { return true; } 1186 virtual bool is_intrinsic() const { return true; } 1187 1188 virtual JVMState* generate(JVMState* jvms); 1189 }; 1190 1191 1192 CallGenerator* CallGenerator::for_predicated_intrinsic(CallGenerator* intrinsic, 1193 CallGenerator* cg) { 1194 return new PredicatedIntrinsicGenerator(intrinsic, cg); 1195 } 1196 1197 1198 JVMState* PredicatedIntrinsicGenerator::generate(JVMState* jvms) { 1199 // The code we want to generate here is: 1200 // if (receiver == nullptr) 1201 // uncommon_Trap 1202 // if (predicate(0)) 1203 // do_intrinsic(0) 1204 // else 1205 // if (predicate(1)) 1206 // do_intrinsic(1) 1207 // ... 1208 // else 1209 // do_java_comp 1210 1211 GraphKit kit(jvms); 1212 PhaseGVN& gvn = kit.gvn(); 1213 1214 CompileLog* log = kit.C->log(); 1215 if (log != nullptr) { 1216 log->elem("predicated_intrinsic bci='%d' method='%d'", 1217 jvms->bci(), log->identify(method())); 1218 } 1219 1220 if (!method()->is_static()) { 1221 // We need an explicit receiver null_check before checking its type in predicate. 1222 // We share a map with the caller, so his JVMS gets adjusted. 1223 Node* receiver = kit.null_check_receiver_before_call(method()); 1224 if (kit.stopped()) { 1225 return kit.transfer_exceptions_into_jvms(); 1226 } 1227 } 1228 1229 int n_predicates = _intrinsic->predicates_count(); 1230 assert(n_predicates > 0, "sanity"); 1231 1232 JVMState** result_jvms = NEW_RESOURCE_ARRAY(JVMState*, (n_predicates+1)); 1233 1234 // Region for normal compilation code if intrinsic failed. 1235 Node* slow_region = new RegionNode(1); 1236 1237 int results = 0; 1238 for (int predicate = 0; (predicate < n_predicates) && !kit.stopped(); predicate++) { 1239 #ifdef ASSERT 1240 JVMState* old_jvms = kit.jvms(); 1241 SafePointNode* old_map = kit.map(); 1242 Node* old_io = old_map->i_o(); 1243 Node* old_mem = old_map->memory(); 1244 Node* old_exc = old_map->next_exception(); 1245 #endif 1246 Node* else_ctrl = _intrinsic->generate_predicate(kit.sync_jvms(), predicate); 1247 #ifdef ASSERT 1248 // Assert(no_new_memory && no_new_io && no_new_exceptions) after generate_predicate. 1249 assert(old_jvms == kit.jvms(), "generate_predicate should not change jvm state"); 1250 SafePointNode* new_map = kit.map(); 1251 assert(old_io == new_map->i_o(), "generate_predicate should not change i_o"); 1252 assert(old_mem == new_map->memory(), "generate_predicate should not change memory"); 1253 assert(old_exc == new_map->next_exception(), "generate_predicate should not add exceptions"); 1254 #endif 1255 if (!kit.stopped()) { 1256 PreserveJVMState pjvms(&kit); 1257 // Generate intrinsic code: 1258 assert(_intrinsic->is_inline(), "LibraryIntrinsic"); 1259 JVMState* new_jvms = _intrinsic->generate(kit.sync_jvms()); 1260 if (new_jvms == nullptr) { 1261 // Intrinsic failed, use normal compilation path for this predicate. 1262 slow_region->add_req(kit.control()); 1263 } else { 1264 kit.add_exception_states_from(new_jvms); 1265 kit.set_jvms(new_jvms); 1266 if (!kit.stopped()) { 1267 result_jvms[results++] = kit.jvms(); 1268 } 1269 } 1270 } 1271 if (else_ctrl == nullptr) { 1272 else_ctrl = kit.C->top(); 1273 } 1274 kit.set_control(else_ctrl); 1275 } 1276 if (!kit.stopped()) { 1277 // Final 'else' after predicates. 1278 slow_region->add_req(kit.control()); 1279 } 1280 if (slow_region->req() > 1) { 1281 PreserveJVMState pjvms(&kit); 1282 // Generate normal compilation code: 1283 kit.set_control(gvn.transform(slow_region)); 1284 JVMState* new_jvms = _cg->generate(kit.sync_jvms()); 1285 if (kit.failing()) 1286 return nullptr; // might happen because of NodeCountInliningCutoff 1287 assert(new_jvms != nullptr, "must be"); 1288 kit.add_exception_states_from(new_jvms); 1289 kit.set_jvms(new_jvms); 1290 if (!kit.stopped()) { 1291 result_jvms[results++] = kit.jvms(); 1292 } 1293 } 1294 1295 if (results == 0) { 1296 // All paths ended in uncommon traps. 1297 (void) kit.stop(); 1298 return kit.transfer_exceptions_into_jvms(); 1299 } 1300 1301 if (results == 1) { // Only one path 1302 kit.set_jvms(result_jvms[0]); 1303 return kit.transfer_exceptions_into_jvms(); 1304 } 1305 1306 // Merge all paths. 1307 kit.C->set_has_split_ifs(true); // Has chance for split-if optimization 1308 RegionNode* region = new RegionNode(results + 1); 1309 Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO); 1310 for (int i = 0; i < results; i++) { 1311 JVMState* jvms = result_jvms[i]; 1312 int path = i + 1; 1313 SafePointNode* map = jvms->map(); 1314 region->init_req(path, map->control()); 1315 iophi->set_req(path, map->i_o()); 1316 if (i == 0) { 1317 kit.set_jvms(jvms); 1318 } else { 1319 kit.merge_memory(map->merged_memory(), region, path); 1320 } 1321 } 1322 kit.set_control(gvn.transform(region)); 1323 kit.set_i_o(gvn.transform(iophi)); 1324 // Transform new memory Phis. 1325 for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) { 1326 Node* phi = mms.memory(); 1327 if (phi->is_Phi() && phi->in(0) == region) { 1328 mms.set_memory(gvn.transform(phi)); 1329 } 1330 } 1331 1332 // Merge debug info. 1333 Node** ins = NEW_RESOURCE_ARRAY(Node*, results); 1334 uint tos = kit.jvms()->stkoff() + kit.sp(); 1335 Node* map = kit.map(); 1336 uint limit = map->req(); 1337 for (uint i = TypeFunc::Parms; i < limit; i++) { 1338 // Skip unused stack slots; fast forward to monoff(); 1339 if (i == tos) { 1340 i = kit.jvms()->monoff(); 1341 if( i >= limit ) break; 1342 } 1343 Node* n = map->in(i); 1344 ins[0] = n; 1345 const Type* t = gvn.type(n); 1346 bool needs_phi = false; 1347 for (int j = 1; j < results; j++) { 1348 JVMState* jvms = result_jvms[j]; 1349 Node* jmap = jvms->map(); 1350 Node* m = nullptr; 1351 if (jmap->req() > i) { 1352 m = jmap->in(i); 1353 if (m != n) { 1354 needs_phi = true; 1355 t = t->meet_speculative(gvn.type(m)); 1356 } 1357 } 1358 ins[j] = m; 1359 } 1360 if (needs_phi) { 1361 Node* phi = PhiNode::make(region, n, t); 1362 for (int j = 1; j < results; j++) { 1363 phi->set_req(j + 1, ins[j]); 1364 } 1365 map->set_req(i, gvn.transform(phi)); 1366 } 1367 } 1368 1369 return kit.transfer_exceptions_into_jvms(); 1370 } 1371 1372 //-------------------------UncommonTrapCallGenerator----------------------------- 1373 // Internal class which handles all out-of-line calls checking receiver type. 1374 class UncommonTrapCallGenerator : public CallGenerator { 1375 Deoptimization::DeoptReason _reason; 1376 Deoptimization::DeoptAction _action; 1377 1378 public: 1379 UncommonTrapCallGenerator(ciMethod* m, 1380 Deoptimization::DeoptReason reason, 1381 Deoptimization::DeoptAction action) 1382 : CallGenerator(m) 1383 { 1384 _reason = reason; 1385 _action = action; 1386 } 1387 1388 virtual bool is_virtual() const { ShouldNotReachHere(); return false; } 1389 virtual bool is_trap() const { return true; } 1390 1391 virtual JVMState* generate(JVMState* jvms); 1392 }; 1393 1394 1395 CallGenerator* 1396 CallGenerator::for_uncommon_trap(ciMethod* m, 1397 Deoptimization::DeoptReason reason, 1398 Deoptimization::DeoptAction action) { 1399 return new UncommonTrapCallGenerator(m, reason, action); 1400 } 1401 1402 1403 JVMState* UncommonTrapCallGenerator::generate(JVMState* jvms) { 1404 GraphKit kit(jvms); 1405 kit.C->print_inlining_update(this); 1406 // Take the trap with arguments pushed on the stack. (Cf. null_check_receiver). 1407 // Callsite signature can be different from actual method being called (i.e _linkTo* sites). 1408 // Use callsite signature always. 1409 ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci()); 1410 int nargs = declared_method->arg_size(); 1411 kit.inc_sp(nargs); 1412 assert(nargs <= kit.sp() && kit.sp() <= jvms->stk_size(), "sane sp w/ args pushed"); 1413 if (_reason == Deoptimization::Reason_class_check && 1414 _action == Deoptimization::Action_maybe_recompile) { 1415 // Temp fix for 6529811 1416 // Don't allow uncommon_trap to override our decision to recompile in the event 1417 // of a class cast failure for a monomorphic call as it will never let us convert 1418 // the call to either bi-morphic or megamorphic and can lead to unc-trap loops 1419 bool keep_exact_action = true; 1420 kit.uncommon_trap(_reason, _action, nullptr, "monomorphic vcall checkcast", false, keep_exact_action); 1421 } else { 1422 kit.uncommon_trap(_reason, _action); 1423 } 1424 return kit.transfer_exceptions_into_jvms(); 1425 } 1426 1427 // (Note: Moved hook_up_call to GraphKit::set_edges_for_java_call.) 1428 1429 // (Node: Merged hook_up_exits into ParseGenerator::generate.)