12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "ci/bcEscapeAnalyzer.hpp"
26 #include "ci/ciCallSite.hpp"
27 #include "ci/ciObjArray.hpp"
28 #include "ci/ciMemberName.hpp"
29 #include "ci/ciMethodHandle.hpp"
30 #include "classfile/javaClasses.hpp"
31 #include "compiler/compileLog.hpp"
32 #include "opto/addnode.hpp"
33 #include "opto/callGenerator.hpp"
34 #include "opto/callnode.hpp"
35 #include "opto/castnode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/parse.hpp"
38 #include "opto/rootnode.hpp"
39 #include "opto/runtime.hpp"
40 #include "opto/subnode.hpp"
41 #include "runtime/os.inline.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "utilities/debug.hpp"
44
45 // Utility function.
46 const TypeFunc* CallGenerator::tf() const {
47 return TypeFunc::make(method());
48 }
49
50 bool CallGenerator::is_inlined_method_handle_intrinsic(JVMState* jvms, ciMethod* m) {
51 return is_inlined_method_handle_intrinsic(jvms->method(), jvms->bci(), m);
52 }
53
54 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* caller, int bci, ciMethod* m) {
55 ciMethod* symbolic_info = caller->get_method_at_bci(bci);
56 return is_inlined_method_handle_intrinsic(symbolic_info, m);
101 GraphKit& exits = parser.exits();
102
103 if (C->failing()) {
104 while (exits.pop_exception_state() != nullptr) ;
105 return nullptr;
106 }
107
108 assert(exits.jvms()->same_calls_as(jvms), "sanity");
109
110 // Simply return the exit state of the parser,
111 // augmented by any exceptional states.
112 return exits.transfer_exceptions_into_jvms();
113 }
114
115 //---------------------------DirectCallGenerator------------------------------
116 // Internal class which handles all out-of-line calls w/o receiver type checks.
117 class DirectCallGenerator : public CallGenerator {
118 private:
119 CallStaticJavaNode* _call_node;
120 // Force separate memory and I/O projections for the exceptional
121 // paths to facilitate late inlinig.
122 bool _separate_io_proj;
123
124 protected:
125 void set_call_node(CallStaticJavaNode* call) { _call_node = call; }
126
127 public:
128 DirectCallGenerator(ciMethod* method, bool separate_io_proj)
129 : CallGenerator(method),
130 _separate_io_proj(separate_io_proj)
131 {
132 }
133 virtual JVMState* generate(JVMState* jvms);
134
135 virtual CallNode* call_node() const { return _call_node; }
136 virtual CallGenerator* with_call_node(CallNode* call) {
137 DirectCallGenerator* dcg = new DirectCallGenerator(method(), _separate_io_proj);
138 dcg->set_call_node(call->as_CallStaticJava());
139 return dcg;
140 }
141 };
142
143 JVMState* DirectCallGenerator::generate(JVMState* jvms) {
144 GraphKit kit(jvms);
145 bool is_static = method()->is_static();
146 address target = is_static ? SharedRuntime::get_resolve_static_call_stub()
147 : SharedRuntime::get_resolve_opt_virtual_call_stub();
148
149 if (kit.C->log() != nullptr) {
150 kit.C->log()->elem("direct_call bci='%d'", jvms->bci());
151 }
152
153 CallStaticJavaNode* call = new CallStaticJavaNode(kit.C, tf(), target, method());
154 if (is_inlined_method_handle_intrinsic(jvms, method())) {
155 // To be able to issue a direct call and skip a call to MH.linkTo*/invokeBasic adapter,
156 // additional information about the method being invoked should be attached
157 // to the call site to make resolution logic work
158 // (see SharedRuntime::resolve_static_call_C).
159 call->set_override_symbolic_info(true);
160 }
161 _call_node = call; // Save the call node in case we need it later
162 if (!is_static) {
163 // Make an explicit receiver null_check as part of this call.
164 // Since we share a map with the caller, his JVMS gets adjusted.
165 kit.null_check_receiver_before_call(method());
166 if (kit.stopped()) {
167 // And dump it back to the caller, decorated with any exceptions:
168 return kit.transfer_exceptions_into_jvms();
169 }
170 // Mark the call node as virtual, sort of:
171 call->set_optimized_virtual(true);
172 if (method()->is_method_handle_intrinsic() ||
173 method()->is_compiled_lambda_form()) {
174 call->set_method_handle_invoke(true);
175 }
176 }
177 kit.set_arguments_for_java_call(call);
178 kit.set_edges_for_java_call(call, false, _separate_io_proj);
179 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
180 kit.push_node(method()->return_type()->basic_type(), ret);
181 return kit.transfer_exceptions_into_jvms();
182 }
183
184 //--------------------------VirtualCallGenerator------------------------------
185 // Internal class which handles all out-of-line calls checking receiver type.
186 class VirtualCallGenerator : public CallGenerator {
187 private:
188 int _vtable_index;
189 bool _separate_io_proj;
190 CallDynamicJavaNode* _call_node;
191
192 protected:
193 void set_call_node(CallDynamicJavaNode* call) { _call_node = call; }
194
195 public:
196 VirtualCallGenerator(ciMethod* method, int vtable_index, bool separate_io_proj)
197 : CallGenerator(method), _vtable_index(vtable_index), _separate_io_proj(separate_io_proj), _call_node(nullptr)
198 {
199 assert(vtable_index == Method::invalid_vtable_index ||
200 vtable_index >= 0, "either invalid or usable");
201 }
202 virtual bool is_virtual() const { return true; }
203 virtual JVMState* generate(JVMState* jvms);
204
205 virtual CallNode* call_node() const { return _call_node; }
206 int vtable_index() const { return _vtable_index; }
207
208 virtual CallGenerator* with_call_node(CallNode* call) {
209 VirtualCallGenerator* cg = new VirtualCallGenerator(method(), _vtable_index, _separate_io_proj);
210 cg->set_call_node(call->as_CallDynamicJava());
211 return cg;
212 }
213 };
214
215 JVMState* VirtualCallGenerator::generate(JVMState* jvms) {
216 GraphKit kit(jvms);
217 Node* receiver = kit.argument(0);
218
219 if (kit.C->log() != nullptr) {
220 kit.C->log()->elem("virtual_call bci='%d'", jvms->bci());
221 }
222
223 // If the receiver is a constant null, do not torture the system
224 // by attempting to call through it. The compile will proceed
225 // correctly, but may bail out in final_graph_reshaping, because
226 // the call instruction will have a seemingly deficient out-count.
227 // (The bailout says something misleading about an "infinite loop".)
228 if (kit.gvn().type(receiver)->higher_equal(TypePtr::NULL_PTR)) {
229 assert(Bytecodes::is_invoke(kit.java_bc()), "%d: %s", kit.java_bc(), Bytecodes::name(kit.java_bc()));
230 ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci());
231 int arg_size = declared_method->signature()->arg_size_for_bc(kit.java_bc());
232 kit.inc_sp(arg_size); // restore arguments
233 kit.uncommon_trap(Deoptimization::Reason_null_check,
234 Deoptimization::Action_none,
235 nullptr, "null receiver");
236 return kit.transfer_exceptions_into_jvms();
237 }
238
256 }
257
258 assert(!method()->is_static(), "virtual call must not be to static");
259 assert(!method()->is_final(), "virtual call should not be to final");
260 assert(!method()->is_private(), "virtual call should not be to private");
261 assert(_vtable_index == Method::invalid_vtable_index || !UseInlineCaches,
262 "no vtable calls if +UseInlineCaches ");
263 address target = SharedRuntime::get_resolve_virtual_call_stub();
264 // Normal inline cache used for call
265 CallDynamicJavaNode* call = new CallDynamicJavaNode(tf(), target, method(), _vtable_index);
266 if (is_inlined_method_handle_intrinsic(jvms, method())) {
267 // To be able to issue a direct call (optimized virtual or virtual)
268 // and skip a call to MH.linkTo*/invokeBasic adapter, additional information
269 // about the method being invoked should be attached to the call site to
270 // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C).
271 call->set_override_symbolic_info(true);
272 }
273 _call_node = call; // Save the call node in case we need it later
274
275 kit.set_arguments_for_java_call(call);
276 kit.set_edges_for_java_call(call, false /*must_throw*/, _separate_io_proj);
277 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
278 kit.push_node(method()->return_type()->basic_type(), ret);
279
280 // Represent the effect of an implicit receiver null_check
281 // as part of this call. Since we share a map with the caller,
282 // his JVMS gets adjusted.
283 kit.cast_not_null(receiver);
284 return kit.transfer_exceptions_into_jvms();
285 }
286
287 CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) {
288 if (InlineTree::check_can_parse(m) != nullptr) return nullptr;
289 return new ParseGenerator(m, expected_uses);
290 }
291
292 // As a special case, the JVMS passed to this CallGenerator is
293 // for the method execution already in progress, not just the JVMS
294 // of the caller. Thus, this CallGenerator cannot be mixed with others!
295 CallGenerator* CallGenerator::for_osr(ciMethod* m, int osr_bci) {
340 // parse is finished.
341 if (!is_mh_late_inline()) {
342 C->add_late_inline(this);
343 }
344
345 // Emit the CallStaticJava and request separate projections so
346 // that the late inlining logic can distinguish between fall
347 // through and exceptional uses of the memory and io projections
348 // as is done for allocations and macro expansion.
349 return DirectCallGenerator::generate(jvms);
350 }
351
352 virtual void set_unique_id(jlong id) {
353 _unique_id = id;
354 }
355
356 virtual jlong unique_id() const {
357 return _unique_id;
358 }
359
360 virtual CallGenerator* with_call_node(CallNode* call) {
361 LateInlineCallGenerator* cg = new LateInlineCallGenerator(method(), _inline_cg, _is_pure_call);
362 cg->set_call_node(call->as_CallStaticJava());
363 return cg;
364 }
365 };
366
367 CallGenerator* CallGenerator::for_late_inline(ciMethod* method, CallGenerator* inline_cg) {
368 return new LateInlineCallGenerator(method, inline_cg);
369 }
370
371 class LateInlineMHCallGenerator : public LateInlineCallGenerator {
372 ciMethod* _caller;
373 bool _input_not_const;
374
375 virtual bool do_late_inline_check(Compile* C, JVMState* jvms);
376
377 public:
378 LateInlineMHCallGenerator(ciMethod* caller, ciMethod* callee, bool input_not_const) :
379 LateInlineCallGenerator(callee, nullptr), _caller(caller), _input_not_const(input_not_const) {}
401 cg->set_call_node(call->as_CallStaticJava());
402 return cg;
403 }
404 };
405
406 bool LateInlineMHCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) {
407 // When inlining a virtual call, the null check at the call and the call itself can throw. These 2 paths have different
408 // expression stacks which causes late inlining to break. The MH invoker is not expected to be called from a method with
409 // exception handlers. When there is no exception handler, GraphKit::builtin_throw() pops the stack which solves the issue
410 // of late inlining with exceptions.
411 assert(!jvms->method()->has_exception_handlers() ||
412 (method()->intrinsic_id() != vmIntrinsics::_linkToVirtual &&
413 method()->intrinsic_id() != vmIntrinsics::_linkToInterface), "no exception handler expected");
414 // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call.
415 bool allow_inline = C->inlining_incrementally();
416 bool input_not_const = true;
417 CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), allow_inline, input_not_const);
418 assert(!input_not_const, "sanity"); // shouldn't have been scheduled for inlining in the first place
419
420 if (cg != nullptr) {
421 if (!allow_inline) {
422 C->inline_printer()->record(cg->method(), call_node()->jvms(), InliningResult::FAILURE,
423 "late method handle call resolution");
424 }
425 assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline || StressIncrementalInlining, "we're doing late inlining");
426 _inline_cg = cg;
427 C->dec_number_of_mh_late_inlines();
428 return true;
429 } else {
430 // Method handle call which has a constant appendix argument should be either inlined or replaced with a direct call
431 // unless there's a signature mismatch between caller and callee. If the failure occurs, there's not much to be improved later,
432 // so don't reinstall the generator to avoid pushing the generator between IGVN and incremental inlining indefinitely.
433 return false;
434 }
435 }
436
437 CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const) {
438 assert(IncrementalInlineMH, "required");
439 Compile::current()->inc_number_of_mh_late_inlines();
440 CallGenerator* cg = new LateInlineMHCallGenerator(caller, callee, input_not_const);
557
558 void LateInlineMHCallGenerator::do_late_inline() {
559 CallGenerator::do_late_inline_helper();
560 }
561
562 void LateInlineVirtualCallGenerator::do_late_inline() {
563 assert(_callee != nullptr, "required"); // set up in CallDynamicJavaNode::Ideal
564 CallGenerator::do_late_inline_helper();
565 }
566
567 void CallGenerator::do_late_inline_helper() {
568 assert(is_late_inline(), "only late inline allowed");
569
570 // Can't inline it
571 CallNode* call = call_node();
572 if (call == nullptr || call->outcnt() == 0 ||
573 call->in(0) == nullptr || call->in(0)->is_top()) {
574 return;
575 }
576
577 const TypeTuple *r = call->tf()->domain();
578 for (int i1 = 0; i1 < method()->arg_size(); i1++) {
579 if (call->in(TypeFunc::Parms + i1)->is_top() && r->field_at(TypeFunc::Parms + i1) != Type::HALF) {
580 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
581 return;
582 }
583 }
584
585 if (call->in(TypeFunc::Memory)->is_top()) {
586 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
587 return;
588 }
589 if (call->in(TypeFunc::Memory)->is_MergeMem()) {
590 MergeMemNode* merge_mem = call->in(TypeFunc::Memory)->as_MergeMem();
591 if (merge_mem->base_memory() == merge_mem->empty_memory()) {
592 return; // dead path
593 }
594 }
595
596 // check for unreachable loop
597 CallProjections callprojs;
598 // Similar to incremental inlining, don't assert that all call
599 // projections are still there for post-parse call devirtualization.
600 bool do_asserts = !is_mh_late_inline() && !is_virtual_late_inline();
601 call->extract_projections(&callprojs, true, do_asserts);
602 if ((callprojs.fallthrough_catchproj == call->in(0)) ||
603 (callprojs.catchall_catchproj == call->in(0)) ||
604 (callprojs.fallthrough_memproj == call->in(TypeFunc::Memory)) ||
605 (callprojs.catchall_memproj == call->in(TypeFunc::Memory)) ||
606 (callprojs.fallthrough_ioproj == call->in(TypeFunc::I_O)) ||
607 (callprojs.catchall_ioproj == call->in(TypeFunc::I_O)) ||
608 (callprojs.resproj != nullptr && call->find_edge(callprojs.resproj) != -1) ||
609 (callprojs.exobj != nullptr && call->find_edge(callprojs.exobj) != -1)) {
610 return;
611 }
612
613 Compile* C = Compile::current();
614 // Remove inlined methods from Compiler's lists.
615 if (call->is_macro()) {
616 C->remove_macro_node(call);
617 }
618
619 // The call is marked as pure (no important side effects), but result isn't used.
620 // It's safe to remove the call.
621 bool result_not_used = (callprojs.resproj == nullptr || callprojs.resproj->outcnt() == 0);
622
623 if (is_pure_call() && result_not_used) {
624 GraphKit kit(call->jvms());
625 kit.replace_call(call, C->top(), true, do_asserts);
626 } else {
627 // Make a clone of the JVMState that appropriate to use for driving a parse
628 JVMState* old_jvms = call->jvms();
629 JVMState* jvms = old_jvms->clone_shallow(C);
630 uint size = call->req();
631 SafePointNode* map = new SafePointNode(size, jvms);
632 for (uint i1 = 0; i1 < size; i1++) {
633 map->init_req(i1, call->in(i1));
634 }
635
636 // Make sure the state is a MergeMem for parsing.
637 if (!map->in(TypeFunc::Memory)->is_MergeMem()) {
638 Node* mem = MergeMemNode::make(map->in(TypeFunc::Memory));
639 C->initial_gvn()->set_type_bottom(mem);
640 map->set_req(TypeFunc::Memory, mem);
641 }
642
643 uint nargs = method()->arg_size();
644 // blow away old call arguments
645 Node* top = C->top();
646 for (uint i1 = 0; i1 < nargs; i1++) {
647 map->set_req(TypeFunc::Parms + i1, top);
648 }
649 jvms->set_map(map);
650
651 // Make enough space in the expression stack to transfer
652 // the incoming arguments and return value.
653 map->ensure_stack(jvms, jvms->method()->max_stack());
654 for (uint i1 = 0; i1 < nargs; i1++) {
655 map->set_argument(jvms, i1, call->in(TypeFunc::Parms + i1));
656 }
657
658 C->log_late_inline(this);
659
660 // JVMState is ready, so time to perform some checks and prepare for inlining attempt.
661 if (!do_late_inline_check(C, jvms)) {
662 map->disconnect_inputs(C);
663 return;
664 }
665
666 // Setup default node notes to be picked up by the inlining
667 Node_Notes* old_nn = C->node_notes_at(call->_idx);
668 if (old_nn != nullptr) {
669 Node_Notes* entry_nn = old_nn->clone(C);
670 entry_nn->set_jvms(jvms);
671 C->set_default_node_notes(entry_nn);
672 }
673
674 // Now perform the inlining using the synthesized JVMState
675 JVMState* new_jvms = inline_cg()->generate(jvms);
676 if (new_jvms == nullptr) return; // no change
677 if (C->failing()) return;
678
679 if (is_mh_late_inline()) {
680 C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (method handle)");
681 } else if (is_string_late_inline()) {
682 C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (string method)");
683 } else if (is_boxing_late_inline()) {
684 C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (boxing method)");
685 } else if (is_vector_reboxing_late_inline()) {
686 C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (vector reboxing method)");
687 } else {
688 C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded");
689 }
690
691 // Capture any exceptional control flow
692 GraphKit kit(new_jvms);
693
694 // Find the result object
695 Node* result = C->top();
696 int result_size = method()->return_type()->size();
697 if (result_size != 0 && !kit.stopped()) {
698 result = (result_size == 1) ? kit.pop() : kit.pop_pair();
699 }
700
701 if (call->is_CallStaticJava() && call->as_CallStaticJava()->is_boxing_method()) {
702 result = kit.must_be_not_null(result, false);
703 }
704
705 if (inline_cg()->is_inline()) {
706 C->set_has_loops(C->has_loops() || inline_cg()->method()->has_loops());
707 C->env()->notice_inlined_method(inline_cg()->method());
708 }
709 C->set_inlining_progress(true);
710 C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup
711 kit.replace_call(call, result, true, do_asserts);
712 }
713 }
714
715 class LateInlineStringCallGenerator : public LateInlineCallGenerator {
716
717 public:
718 LateInlineStringCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
719 LateInlineCallGenerator(method, inline_cg) {}
720
721 virtual JVMState* generate(JVMState* jvms) {
722 Compile *C = Compile::current();
723
724 C->log_inline_id(this);
725
726 C->add_string_late_inline(this);
727
728 JVMState* new_jvms = DirectCallGenerator::generate(jvms);
729 return new_jvms;
730 }
919 // Inline failed, so make a direct call.
920 assert(_if_hit->is_inline(), "must have been a failed inline");
921 CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method());
922 new_jvms = cg->generate(kit.sync_jvms());
923 }
924 kit.add_exception_states_from(new_jvms);
925 kit.set_jvms(new_jvms);
926
927 // Need to merge slow and fast?
928 if (slow_map == nullptr) {
929 // The fast path is the only path remaining.
930 return kit.transfer_exceptions_into_jvms();
931 }
932
933 if (kit.stopped()) {
934 // Inlined method threw an exception, so it's just the slow path after all.
935 kit.set_jvms(slow_jvms);
936 return kit.transfer_exceptions_into_jvms();
937 }
938
939 // There are 2 branches and the replaced nodes are only valid on
940 // one: restore the replaced nodes to what they were before the
941 // branch.
942 kit.map()->set_replaced_nodes(replaced_nodes);
943
944 // Finish the diamond.
945 kit.C->set_has_split_ifs(true); // Has chance for split-if optimization
946 RegionNode* region = new RegionNode(3);
947 region->init_req(1, kit.control());
948 region->init_req(2, slow_map->control());
949 kit.set_control(gvn.transform(region));
950 Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO);
951 iophi->set_req(2, slow_map->i_o());
952 kit.set_i_o(gvn.transform(iophi));
953 // Merge memory
954 kit.merge_memory(slow_map->merged_memory(), region, 2);
955 // Transform new memory Phis.
956 for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) {
957 Node* phi = mms.memory();
958 if (phi->is_Phi() && phi->in(0) == region) {
959 mms.set_memory(gvn.transform(phi));
960 }
961 }
962 uint tos = kit.jvms()->stkoff() + kit.sp();
963 uint limit = slow_map->req();
964 for (uint i = TypeFunc::Parms; i < limit; i++) {
965 // Skip unused stack slots; fast forward to monoff();
966 if (i == tos) {
967 i = kit.jvms()->monoff();
968 if( i >= limit ) break;
969 }
970 Node* m = kit.map()->in(i);
971 Node* n = slow_map->in(i);
972 if (m != n) {
973 const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
974 Node* phi = PhiNode::make(region, m, t);
975 phi->set_req(2, n);
976 kit.map()->set_req(i, gvn.transform(phi));
977 }
978 }
979 return kit.transfer_exceptions_into_jvms();
980 }
981
982
983 CallGenerator* CallGenerator::for_method_handle_call(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline) {
984 assert(callee->is_method_handle_intrinsic(), "for_method_handle_call mismatch");
985 bool input_not_const;
986 CallGenerator* cg = CallGenerator::for_method_handle_inline(jvms, caller, callee, allow_inline, input_not_const);
987 Compile* C = Compile::current();
988 bool should_delay = C->should_delay_inlining();
989 if (cg != nullptr) {
990 if (should_delay && IncrementalInlineMH) {
991 return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
992 } else {
993 return cg;
994 }
995 }
996 int bci = jvms->bci();
997 ciCallProfile profile = caller->call_profile_at_bci(bci);
998 int call_site_count = caller->scale_count(profile.count());
999
1000 if (IncrementalInlineMH && call_site_count > 0 &&
1001 (should_delay || input_not_const || !C->inlining_incrementally() || C->over_inlining_cutoff())) {
1002 return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
1003 } else {
1004 // Out-of-line call.
1005 return CallGenerator::for_direct_call(callee);
1006 }
1007 }
1008
1009 CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline, bool& input_not_const) {
1010 GraphKit kit(jvms);
1011 PhaseGVN& gvn = kit.gvn();
1012 Compile* C = kit.C;
1013 vmIntrinsics::ID iid = callee->intrinsic_id();
1014 input_not_const = true;
1015 if (StressMethodHandleLinkerInlining) {
1016 allow_inline = false;
1017 }
1018 switch (iid) {
1019 case vmIntrinsics::_invokeBasic:
1020 {
1021 // Get MethodHandle receiver:
1022 Node* receiver = kit.argument(0);
1023 if (receiver->Opcode() == Op_ConP) {
1024 input_not_const = false;
1025 const TypeOopPtr* recv_toop = receiver->bottom_type()->isa_oopptr();
1026 if (recv_toop != nullptr) {
1027 ciMethod* target = recv_toop->const_oop()->as_method_handle()->get_vmtarget();
1028 const int vtable_index = Method::invalid_vtable_index;
1036 false /* call_does_dispatch */,
1037 jvms,
1038 allow_inline,
1039 PROB_ALWAYS);
1040 return cg;
1041 } else {
1042 assert(receiver->bottom_type() == TypePtr::NULL_PTR, "not a null: %s",
1043 Type::str(receiver->bottom_type()));
1044 print_inlining_failure(C, callee, jvms, "receiver is always null");
1045 }
1046 } else {
1047 print_inlining_failure(C, callee, jvms, "receiver not constant");
1048 }
1049 } break;
1050
1051 case vmIntrinsics::_linkToVirtual:
1052 case vmIntrinsics::_linkToStatic:
1053 case vmIntrinsics::_linkToSpecial:
1054 case vmIntrinsics::_linkToInterface:
1055 {
1056 // Get MemberName argument:
1057 Node* member_name = kit.argument(callee->arg_size() - 1);
1058 if (member_name->Opcode() == Op_ConP) {
1059 input_not_const = false;
1060 const TypeOopPtr* oop_ptr = member_name->bottom_type()->is_oopptr();
1061 ciMethod* target = oop_ptr->const_oop()->as_member_name()->get_vmtarget();
1062
1063 if (!ciMethod::is_consistent_info(callee, target)) {
1064 print_inlining_failure(C, callee, jvms, "signatures mismatch");
1065 return nullptr;
1066 }
1067
1068 // In lambda forms we erase signature types to avoid resolving issues
1069 // involving class loaders. When we optimize a method handle invoke
1070 // to a direct call we must cast the receiver and arguments to its
1071 // actual types.
1072 ciSignature* signature = target->signature();
1073 const int receiver_skip = target->is_static() ? 0 : 1;
1074 // Cast receiver to its type.
1075 if (!target->is_static()) {
1076 Node* recv = kit.argument(0);
1077 Node* casted_recv = kit.maybe_narrow_object_type(recv, signature->accessing_klass());
1107 ciKlass* speculative_receiver_type = nullptr;
1108 if (is_virtual_or_interface) {
1109 ciInstanceKlass* klass = target->holder();
1110 Node* receiver_node = kit.argument(0);
1111 const TypeOopPtr* receiver_type = gvn.type(receiver_node)->isa_oopptr();
1112 // call_does_dispatch and vtable_index are out-parameters. They might be changed.
1113 // optimize_virtual_call() takes 2 different holder
1114 // arguments for a corner case that doesn't apply here (see
1115 // Parse::do_call())
1116 target = C->optimize_virtual_call(caller, klass, klass,
1117 target, receiver_type, is_virtual,
1118 call_does_dispatch, vtable_index, // out-parameters
1119 false /* check_access */);
1120 // We lack profiling at this call but type speculation may
1121 // provide us with a type
1122 speculative_receiver_type = (receiver_type != nullptr) ? receiver_type->speculative_type() : nullptr;
1123 }
1124 CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms,
1125 allow_inline,
1126 PROB_ALWAYS,
1127 speculative_receiver_type);
1128 return cg;
1129 } else {
1130 print_inlining_failure(C, callee, jvms, "member_name not constant");
1131 }
1132 } break;
1133
1134 case vmIntrinsics::_linkToNative:
1135 print_inlining_failure(C, callee, jvms, "native call");
1136 break;
1137
1138 default:
1139 fatal("unexpected intrinsic %d: %s", vmIntrinsics::as_int(iid), vmIntrinsics::name_at(iid));
1140 break;
1141 }
1142 return nullptr;
1143 }
1144
1145 //------------------------PredicatedIntrinsicGenerator------------------------------
1146 // Internal class which handles all predicated Intrinsic calls.
1147 class PredicatedIntrinsicGenerator : public CallGenerator {
1179 // do_intrinsic(0)
1180 // else
1181 // if (predicate(1))
1182 // do_intrinsic(1)
1183 // ...
1184 // else
1185 // do_java_comp
1186
1187 GraphKit kit(jvms);
1188 PhaseGVN& gvn = kit.gvn();
1189
1190 CompileLog* log = kit.C->log();
1191 if (log != nullptr) {
1192 log->elem("predicated_intrinsic bci='%d' method='%d'",
1193 jvms->bci(), log->identify(method()));
1194 }
1195
1196 if (!method()->is_static()) {
1197 // We need an explicit receiver null_check before checking its type in predicate.
1198 // We share a map with the caller, so his JVMS gets adjusted.
1199 Node* receiver = kit.null_check_receiver_before_call(method());
1200 if (kit.stopped()) {
1201 return kit.transfer_exceptions_into_jvms();
1202 }
1203 }
1204
1205 int n_predicates = _intrinsic->predicates_count();
1206 assert(n_predicates > 0, "sanity");
1207
1208 JVMState** result_jvms = NEW_RESOURCE_ARRAY(JVMState*, (n_predicates+1));
1209
1210 // Region for normal compilation code if intrinsic failed.
1211 Node* slow_region = new RegionNode(1);
1212
1213 int results = 0;
1214 for (int predicate = 0; (predicate < n_predicates) && !kit.stopped(); predicate++) {
1215 #ifdef ASSERT
1216 JVMState* old_jvms = kit.jvms();
1217 SafePointNode* old_map = kit.map();
1218 Node* old_io = old_map->i_o();
1219 Node* old_mem = old_map->memory();
|
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "ci/bcEscapeAnalyzer.hpp"
26 #include "ci/ciCallSite.hpp"
27 #include "ci/ciObjArray.hpp"
28 #include "ci/ciMemberName.hpp"
29 #include "ci/ciMethodHandle.hpp"
30 #include "classfile/javaClasses.hpp"
31 #include "compiler/compileLog.hpp"
32 #include "oops/accessDecorators.hpp"
33 #include "opto/addnode.hpp"
34 #include "opto/callGenerator.hpp"
35 #include "opto/callnode.hpp"
36 #include "opto/castnode.hpp"
37 #include "opto/cfgnode.hpp"
38 #include "opto/inlinetypenode.hpp"
39 #include "opto/parse.hpp"
40 #include "opto/rootnode.hpp"
41 #include "opto/runtime.hpp"
42 #include "opto/subnode.hpp"
43 #include "runtime/os.inline.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "utilities/debug.hpp"
46
47 // Utility function.
48 const TypeFunc* CallGenerator::tf() const {
49 return TypeFunc::make(method());
50 }
51
52 bool CallGenerator::is_inlined_method_handle_intrinsic(JVMState* jvms, ciMethod* m) {
53 return is_inlined_method_handle_intrinsic(jvms->method(), jvms->bci(), m);
54 }
55
56 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* caller, int bci, ciMethod* m) {
57 ciMethod* symbolic_info = caller->get_method_at_bci(bci);
58 return is_inlined_method_handle_intrinsic(symbolic_info, m);
103 GraphKit& exits = parser.exits();
104
105 if (C->failing()) {
106 while (exits.pop_exception_state() != nullptr) ;
107 return nullptr;
108 }
109
110 assert(exits.jvms()->same_calls_as(jvms), "sanity");
111
112 // Simply return the exit state of the parser,
113 // augmented by any exceptional states.
114 return exits.transfer_exceptions_into_jvms();
115 }
116
117 //---------------------------DirectCallGenerator------------------------------
118 // Internal class which handles all out-of-line calls w/o receiver type checks.
119 class DirectCallGenerator : public CallGenerator {
120 private:
121 CallStaticJavaNode* _call_node;
122 // Force separate memory and I/O projections for the exceptional
123 // paths to facilitate late inlining.
124 bool _separate_io_proj;
125
126 protected:
127 void set_call_node(CallStaticJavaNode* call) { _call_node = call; }
128
129 public:
130 DirectCallGenerator(ciMethod* method, bool separate_io_proj)
131 : CallGenerator(method),
132 _call_node(nullptr),
133 _separate_io_proj(separate_io_proj)
134 {
135 if (InlineTypeReturnedAsFields && method->is_method_handle_intrinsic()) {
136 // If that call has not been optimized by the time optimizations are over,
137 // we'll need to add a call to create an inline type instance from the klass
138 // returned by the call (see PhaseMacroExpand::expand_mh_intrinsic_return).
139 // Separating memory and I/O projections for exceptions is required to
140 // perform that graph transformation.
141 _separate_io_proj = true;
142 }
143 }
144 virtual JVMState* generate(JVMState* jvms);
145
146 virtual CallNode* call_node() const { return _call_node; }
147 virtual CallGenerator* with_call_node(CallNode* call) {
148 DirectCallGenerator* dcg = new DirectCallGenerator(method(), _separate_io_proj);
149 dcg->set_call_node(call->as_CallStaticJava());
150 return dcg;
151 }
152 };
153
154 JVMState* DirectCallGenerator::generate(JVMState* jvms) {
155 GraphKit kit(jvms);
156 PhaseGVN& gvn = kit.gvn();
157 bool is_static = method()->is_static();
158 address target = is_static ? SharedRuntime::get_resolve_static_call_stub()
159 : SharedRuntime::get_resolve_opt_virtual_call_stub();
160
161 if (kit.C->log() != nullptr) {
162 kit.C->log()->elem("direct_call bci='%d'", jvms->bci());
163 }
164
165 CallStaticJavaNode* call = new CallStaticJavaNode(kit.C, tf(), target, method());
166 if (is_inlined_method_handle_intrinsic(jvms, method())) {
167 // To be able to issue a direct call and skip a call to MH.linkTo*/invokeBasic adapter,
168 // additional information about the method being invoked should be attached
169 // to the call site to make resolution logic work
170 // (see SharedRuntime::resolve_static_call_C).
171 call->set_override_symbolic_info(true);
172 }
173 _call_node = call; // Save the call node in case we need it later
174 if (!is_static) {
175 // Make an explicit receiver null_check as part of this call.
176 // Since we share a map with the caller, his JVMS gets adjusted.
177 kit.null_check_receiver_before_call(method());
178 if (kit.stopped()) {
179 // And dump it back to the caller, decorated with any exceptions:
180 return kit.transfer_exceptions_into_jvms();
181 }
182 // Mark the call node as virtual, sort of:
183 call->set_optimized_virtual(true);
184 if (method()->is_method_handle_intrinsic() ||
185 method()->is_compiled_lambda_form()) {
186 call->set_method_handle_invoke(true);
187 }
188 }
189 kit.set_arguments_for_java_call(call, is_late_inline());
190 if (kit.stopped()) {
191 return kit.transfer_exceptions_into_jvms();
192 }
193 kit.set_edges_for_java_call(call, false, _separate_io_proj);
194 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
195 kit.push_node(method()->return_type()->basic_type(), ret);
196 return kit.transfer_exceptions_into_jvms();
197 }
198
199 //--------------------------VirtualCallGenerator------------------------------
200 // Internal class which handles all out-of-line calls checking receiver type.
201 class VirtualCallGenerator : public CallGenerator {
202 private:
203 int _vtable_index;
204 bool _separate_io_proj;
205 CallDynamicJavaNode* _call_node;
206
207 protected:
208 void set_call_node(CallDynamicJavaNode* call) { _call_node = call; }
209
210 public:
211 VirtualCallGenerator(ciMethod* method, int vtable_index, bool separate_io_proj)
212 : CallGenerator(method), _vtable_index(vtable_index), _separate_io_proj(separate_io_proj), _call_node(nullptr)
213 {
214 assert(vtable_index == Method::invalid_vtable_index ||
215 vtable_index >= 0, "either invalid or usable");
216 }
217 virtual bool is_virtual() const { return true; }
218 virtual JVMState* generate(JVMState* jvms);
219
220 virtual CallNode* call_node() const { return _call_node; }
221 int vtable_index() const { return _vtable_index; }
222
223 virtual CallGenerator* with_call_node(CallNode* call) {
224 VirtualCallGenerator* cg = new VirtualCallGenerator(method(), _vtable_index, _separate_io_proj);
225 cg->set_call_node(call->as_CallDynamicJava());
226 return cg;
227 }
228 };
229
230 JVMState* VirtualCallGenerator::generate(JVMState* jvms) {
231 GraphKit kit(jvms);
232 Node* receiver = kit.argument(0);
233 if (kit.C->log() != nullptr) {
234 kit.C->log()->elem("virtual_call bci='%d'", jvms->bci());
235 }
236
237 // If the receiver is a constant null, do not torture the system
238 // by attempting to call through it. The compile will proceed
239 // correctly, but may bail out in final_graph_reshaping, because
240 // the call instruction will have a seemingly deficient out-count.
241 // (The bailout says something misleading about an "infinite loop".)
242 if (kit.gvn().type(receiver)->higher_equal(TypePtr::NULL_PTR)) {
243 assert(Bytecodes::is_invoke(kit.java_bc()), "%d: %s", kit.java_bc(), Bytecodes::name(kit.java_bc()));
244 ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci());
245 int arg_size = declared_method->signature()->arg_size_for_bc(kit.java_bc());
246 kit.inc_sp(arg_size); // restore arguments
247 kit.uncommon_trap(Deoptimization::Reason_null_check,
248 Deoptimization::Action_none,
249 nullptr, "null receiver");
250 return kit.transfer_exceptions_into_jvms();
251 }
252
270 }
271
272 assert(!method()->is_static(), "virtual call must not be to static");
273 assert(!method()->is_final(), "virtual call should not be to final");
274 assert(!method()->is_private(), "virtual call should not be to private");
275 assert(_vtable_index == Method::invalid_vtable_index || !UseInlineCaches,
276 "no vtable calls if +UseInlineCaches ");
277 address target = SharedRuntime::get_resolve_virtual_call_stub();
278 // Normal inline cache used for call
279 CallDynamicJavaNode* call = new CallDynamicJavaNode(tf(), target, method(), _vtable_index);
280 if (is_inlined_method_handle_intrinsic(jvms, method())) {
281 // To be able to issue a direct call (optimized virtual or virtual)
282 // and skip a call to MH.linkTo*/invokeBasic adapter, additional information
283 // about the method being invoked should be attached to the call site to
284 // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C).
285 call->set_override_symbolic_info(true);
286 }
287 _call_node = call; // Save the call node in case we need it later
288
289 kit.set_arguments_for_java_call(call);
290 if (kit.stopped()) {
291 return kit.transfer_exceptions_into_jvms();
292 }
293 kit.set_edges_for_java_call(call, false /*must_throw*/, _separate_io_proj);
294 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
295 kit.push_node(method()->return_type()->basic_type(), ret);
296
297 // Represent the effect of an implicit receiver null_check
298 // as part of this call. Since we share a map with the caller,
299 // his JVMS gets adjusted.
300 kit.cast_not_null(receiver);
301 return kit.transfer_exceptions_into_jvms();
302 }
303
304 CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) {
305 if (InlineTree::check_can_parse(m) != nullptr) return nullptr;
306 return new ParseGenerator(m, expected_uses);
307 }
308
309 // As a special case, the JVMS passed to this CallGenerator is
310 // for the method execution already in progress, not just the JVMS
311 // of the caller. Thus, this CallGenerator cannot be mixed with others!
312 CallGenerator* CallGenerator::for_osr(ciMethod* m, int osr_bci) {
357 // parse is finished.
358 if (!is_mh_late_inline()) {
359 C->add_late_inline(this);
360 }
361
362 // Emit the CallStaticJava and request separate projections so
363 // that the late inlining logic can distinguish between fall
364 // through and exceptional uses of the memory and io projections
365 // as is done for allocations and macro expansion.
366 return DirectCallGenerator::generate(jvms);
367 }
368
369 virtual void set_unique_id(jlong id) {
370 _unique_id = id;
371 }
372
373 virtual jlong unique_id() const {
374 return _unique_id;
375 }
376
377 virtual CallGenerator* inline_cg() {
378 return _inline_cg;
379 }
380
381 virtual CallGenerator* with_call_node(CallNode* call) {
382 LateInlineCallGenerator* cg = new LateInlineCallGenerator(method(), _inline_cg, _is_pure_call);
383 cg->set_call_node(call->as_CallStaticJava());
384 return cg;
385 }
386 };
387
388 CallGenerator* CallGenerator::for_late_inline(ciMethod* method, CallGenerator* inline_cg) {
389 return new LateInlineCallGenerator(method, inline_cg);
390 }
391
392 class LateInlineMHCallGenerator : public LateInlineCallGenerator {
393 ciMethod* _caller;
394 bool _input_not_const;
395
396 virtual bool do_late_inline_check(Compile* C, JVMState* jvms);
397
398 public:
399 LateInlineMHCallGenerator(ciMethod* caller, ciMethod* callee, bool input_not_const) :
400 LateInlineCallGenerator(callee, nullptr), _caller(caller), _input_not_const(input_not_const) {}
422 cg->set_call_node(call->as_CallStaticJava());
423 return cg;
424 }
425 };
426
427 bool LateInlineMHCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) {
428 // When inlining a virtual call, the null check at the call and the call itself can throw. These 2 paths have different
429 // expression stacks which causes late inlining to break. The MH invoker is not expected to be called from a method with
430 // exception handlers. When there is no exception handler, GraphKit::builtin_throw() pops the stack which solves the issue
431 // of late inlining with exceptions.
432 assert(!jvms->method()->has_exception_handlers() ||
433 (method()->intrinsic_id() != vmIntrinsics::_linkToVirtual &&
434 method()->intrinsic_id() != vmIntrinsics::_linkToInterface), "no exception handler expected");
435 // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call.
436 bool allow_inline = C->inlining_incrementally();
437 bool input_not_const = true;
438 CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), allow_inline, input_not_const);
439 assert(!input_not_const, "sanity"); // shouldn't have been scheduled for inlining in the first place
440
441 if (cg != nullptr) {
442 // AlwaysIncrementalInline causes for_method_handle_inline() to
443 // return a LateInlineCallGenerator. Extract the
444 // InlineCallGenerator from it.
445 if (AlwaysIncrementalInline && cg->is_late_inline() && !cg->is_virtual_late_inline()) {
446 cg = cg->inline_cg();
447 assert(cg != nullptr, "inline call generator expected");
448 }
449
450 if (!allow_inline) {
451 C->inline_printer()->record(cg->method(), call_node()->jvms(), InliningResult::FAILURE,
452 "late method handle call resolution");
453 }
454 assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline || StressIncrementalInlining, "we're doing late inlining");
455 _inline_cg = cg;
456 C->dec_number_of_mh_late_inlines();
457 return true;
458 } else {
459 // Method handle call which has a constant appendix argument should be either inlined or replaced with a direct call
460 // unless there's a signature mismatch between caller and callee. If the failure occurs, there's not much to be improved later,
461 // so don't reinstall the generator to avoid pushing the generator between IGVN and incremental inlining indefinitely.
462 return false;
463 }
464 }
465
466 CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const) {
467 assert(IncrementalInlineMH, "required");
468 Compile::current()->inc_number_of_mh_late_inlines();
469 CallGenerator* cg = new LateInlineMHCallGenerator(caller, callee, input_not_const);
586
587 void LateInlineMHCallGenerator::do_late_inline() {
588 CallGenerator::do_late_inline_helper();
589 }
590
591 void LateInlineVirtualCallGenerator::do_late_inline() {
592 assert(_callee != nullptr, "required"); // set up in CallDynamicJavaNode::Ideal
593 CallGenerator::do_late_inline_helper();
594 }
595
596 void CallGenerator::do_late_inline_helper() {
597 assert(is_late_inline(), "only late inline allowed");
598
599 // Can't inline it
600 CallNode* call = call_node();
601 if (call == nullptr || call->outcnt() == 0 ||
602 call->in(0) == nullptr || call->in(0)->is_top()) {
603 return;
604 }
605
606 const TypeTuple* r = call->tf()->domain_cc();
607 for (uint i1 = TypeFunc::Parms; i1 < r->cnt(); i1++) {
608 if (call->in(i1)->is_top() && r->field_at(i1) != Type::HALF) {
609 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
610 return;
611 }
612 }
613
614 if (call->in(TypeFunc::Memory)->is_top()) {
615 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
616 return;
617 }
618 if (call->in(TypeFunc::Memory)->is_MergeMem()) {
619 MergeMemNode* merge_mem = call->in(TypeFunc::Memory)->as_MergeMem();
620 if (merge_mem->base_memory() == merge_mem->empty_memory()) {
621 return; // dead path
622 }
623 }
624
625 // check for unreachable loop
626 // Similar to incremental inlining, don't assert that all call
627 // projections are still there for post-parse call devirtualization.
628 bool do_asserts = !is_mh_late_inline() && !is_virtual_late_inline();
629 CallProjections* callprojs = call->extract_projections(true, do_asserts);
630 if ((callprojs->fallthrough_catchproj == call->in(0)) ||
631 (callprojs->catchall_catchproj == call->in(0)) ||
632 (callprojs->fallthrough_memproj == call->in(TypeFunc::Memory)) ||
633 (callprojs->catchall_memproj == call->in(TypeFunc::Memory)) ||
634 (callprojs->fallthrough_ioproj == call->in(TypeFunc::I_O)) ||
635 (callprojs->catchall_ioproj == call->in(TypeFunc::I_O)) ||
636 (callprojs->exobj != nullptr && call->find_edge(callprojs->exobj) != -1)) {
637 return;
638 }
639
640 Compile* C = Compile::current();
641 // Remove inlined methods from Compiler's lists.
642 if (call->is_macro()) {
643 C->remove_macro_node(call);
644 }
645
646
647 bool result_not_used = true;
648 for (uint i = 0; i < callprojs->nb_resproj; i++) {
649 if (callprojs->resproj[i] != nullptr) {
650 if (callprojs->resproj[i]->outcnt() != 0) {
651 result_not_used = false;
652 }
653 if (call->find_edge(callprojs->resproj[i]) != -1) {
654 return;
655 }
656 }
657 }
658
659 if (is_pure_call() && result_not_used) {
660 // The call is marked as pure (no important side effects), but result isn't used.
661 // It's safe to remove the call.
662 GraphKit kit(call->jvms());
663 kit.replace_call(call, C->top(), true, do_asserts);
664 } else {
665 // Make a clone of the JVMState that appropriate to use for driving a parse
666 JVMState* old_jvms = call->jvms();
667 JVMState* jvms = old_jvms->clone_shallow(C);
668 uint size = call->req();
669 SafePointNode* map = new SafePointNode(size, jvms);
670 for (uint i1 = 0; i1 < size; i1++) {
671 map->init_req(i1, call->in(i1));
672 }
673
674 PhaseGVN& gvn = *C->initial_gvn();
675 // Make sure the state is a MergeMem for parsing.
676 if (!map->in(TypeFunc::Memory)->is_MergeMem()) {
677 Node* mem = MergeMemNode::make(map->in(TypeFunc::Memory));
678 gvn.set_type_bottom(mem);
679 map->set_req(TypeFunc::Memory, mem);
680 }
681
682 // blow away old call arguments
683 for (uint i1 = TypeFunc::Parms; i1 < r->cnt(); i1++) {
684 map->set_req(i1, C->top());
685 }
686 jvms->set_map(map);
687
688 // Make enough space in the expression stack to transfer
689 // the incoming arguments and return value.
690 map->ensure_stack(jvms, jvms->method()->max_stack());
691 const TypeTuple* domain_sig = call->_tf->domain_sig();
692 uint nargs = method()->arg_size();
693 assert(domain_sig->cnt() - TypeFunc::Parms == nargs, "inconsistent signature");
694
695 uint j = TypeFunc::Parms;
696 int arg_num = 0;
697 for (uint i1 = 0; i1 < nargs; i1++) {
698 const Type* t = domain_sig->field_at(TypeFunc::Parms + i1);
699 if (t->is_inlinetypeptr() && !method()->get_Method()->mismatch() && method()->is_scalarized_arg(arg_num)) {
700 // Inline type arguments are not passed by reference: we get an argument per
701 // field of the inline type. Build InlineTypeNodes from the inline type arguments.
702 GraphKit arg_kit(jvms, &gvn);
703 Node* vt = InlineTypeNode::make_from_multi(&arg_kit, call, t->inline_klass(), j, /* in= */ true, /* null_free= */ !t->maybe_null());
704 map->set_control(arg_kit.control());
705 map->set_argument(jvms, i1, vt);
706 } else {
707 map->set_argument(jvms, i1, call->in(j++));
708 }
709 if (t != Type::HALF) {
710 arg_num++;
711 }
712 }
713
714 C->log_late_inline(this);
715
716 // JVMState is ready, so time to perform some checks and prepare for inlining attempt.
717 if (!do_late_inline_check(C, jvms)) {
718 map->disconnect_inputs(C);
719 return;
720 }
721
722 // Check if we are late inlining a method handle call that returns an inline type as fields.
723 Node* buffer_oop = nullptr;
724 ciMethod* inline_method = inline_cg()->method();
725 ciType* return_type = inline_method->return_type();
726 if (!call->tf()->returns_inline_type_as_fields() &&
727 return_type->is_inlinetype() && return_type->as_inline_klass()->can_be_returned_as_fields()) {
728 // Allocate a buffer for the inline type returned as fields because the caller expects an oop return.
729 // Do this before the method handle call in case the buffer allocation triggers deoptimization and
730 // we need to "re-execute" the call in the interpreter (to make sure the call is only executed once).
731 GraphKit arg_kit(jvms, &gvn);
732 {
733 PreserveReexecuteState preexecs(&arg_kit);
734 arg_kit.jvms()->set_should_reexecute(true);
735 arg_kit.inc_sp(nargs);
736 Node* klass_node = arg_kit.makecon(TypeKlassPtr::make(return_type->as_inline_klass()));
737 buffer_oop = arg_kit.new_instance(klass_node, nullptr, nullptr, /* deoptimize_on_exception */ true);
738 }
739 jvms = arg_kit.transfer_exceptions_into_jvms();
740 }
741
742 // Setup default node notes to be picked up by the inlining
743 Node_Notes* old_nn = C->node_notes_at(call->_idx);
744 if (old_nn != nullptr) {
745 Node_Notes* entry_nn = old_nn->clone(C);
746 entry_nn->set_jvms(jvms);
747 C->set_default_node_notes(entry_nn);
748 }
749
750 // Now perform the inlining using the synthesized JVMState
751 JVMState* new_jvms = inline_cg()->generate(jvms);
752 if (new_jvms == nullptr) return; // no change
753 if (C->failing()) return;
754
755 if (is_mh_late_inline()) {
756 C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (method handle)");
757 } else if (is_string_late_inline()) {
758 C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (string method)");
759 } else if (is_boxing_late_inline()) {
760 C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (boxing method)");
761 } else if (is_vector_reboxing_late_inline()) {
762 C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded (vector reboxing method)");
763 } else {
764 C->inline_printer()->record(method(), jvms, InliningResult::SUCCESS, "late inline succeeded");
765 }
766
767 // Capture any exceptional control flow
768 GraphKit kit(new_jvms);
769
770 // Find the result object
771 Node* result = C->top();
772 int result_size = method()->return_type()->size();
773 if (result_size != 0 && !kit.stopped()) {
774 result = (result_size == 1) ? kit.pop() : kit.pop_pair();
775 }
776
777 if (call->is_CallStaticJava() && call->as_CallStaticJava()->is_boxing_method()) {
778 result = kit.must_be_not_null(result, false);
779 }
780
781 if (inline_cg()->is_inline()) {
782 C->set_has_loops(C->has_loops() || inline_method->has_loops());
783 C->env()->notice_inlined_method(inline_method);
784 }
785 C->set_inlining_progress(true);
786 C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup
787
788 // Handle inline type returns
789 InlineTypeNode* vt = result->isa_InlineType();
790 if (vt != nullptr) {
791 if (call->tf()->returns_inline_type_as_fields()) {
792 vt->replace_call_results(&kit, call, C);
793 } else {
794 // Result might still be allocated (for example, if it has been stored to a non-flat field)
795 if (!vt->is_allocated(&kit.gvn())) {
796 assert(buffer_oop != nullptr, "should have allocated a buffer");
797 RegionNode* region = new RegionNode(3);
798
799 // Check if result is null
800 Node* null_ctl = kit.top();
801 kit.null_check_common(vt->get_null_marker(), T_INT, false, &null_ctl);
802 region->init_req(1, null_ctl);
803 PhiNode* oop = PhiNode::make(region, kit.gvn().zerocon(T_OBJECT), TypeInstPtr::make(TypePtr::BotPTR, vt->type()->inline_klass()));
804 Node* init_mem = kit.reset_memory();
805 PhiNode* mem = PhiNode::make(region, init_mem, Type::MEMORY, TypePtr::BOTTOM);
806
807 // Not null, initialize the buffer
808 kit.set_all_memory(init_mem);
809
810 Node* payload_ptr = kit.basic_plus_adr(buffer_oop, kit.gvn().type(vt)->inline_klass()->payload_offset());
811 vt->store_flat(&kit, buffer_oop, payload_ptr, false, true, true, IN_HEAP | MO_UNORDERED);
812 // Do not let stores that initialize this buffer be reordered with a subsequent
813 // store that would make this buffer accessible by other threads.
814 AllocateNode* alloc = AllocateNode::Ideal_allocation(buffer_oop);
815 assert(alloc != nullptr, "must have an allocation node");
816 kit.insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress));
817 region->init_req(2, kit.control());
818 oop->init_req(2, buffer_oop);
819 mem->init_req(2, kit.merged_memory());
820
821 // Update oop input to buffer
822 kit.gvn().hash_delete(vt);
823 vt->set_oop(kit.gvn(), kit.gvn().transform(oop));
824 vt->set_is_buffered(kit.gvn());
825 vt = kit.gvn().transform(vt)->as_InlineType();
826
827 kit.set_control(kit.gvn().transform(region));
828 kit.set_all_memory(kit.gvn().transform(mem));
829 kit.record_for_igvn(region);
830 kit.record_for_igvn(oop);
831 kit.record_for_igvn(mem);
832 }
833 result = vt;
834 }
835 DEBUG_ONLY(buffer_oop = nullptr);
836 } else {
837 assert(result->is_top() || !call->tf()->returns_inline_type_as_fields(), "Unexpected return value");
838 }
839 assert(buffer_oop == nullptr, "unused buffer allocation");
840
841 kit.replace_call(call, result, true, do_asserts);
842 }
843 }
844
845 class LateInlineStringCallGenerator : public LateInlineCallGenerator {
846
847 public:
848 LateInlineStringCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
849 LateInlineCallGenerator(method, inline_cg) {}
850
851 virtual JVMState* generate(JVMState* jvms) {
852 Compile *C = Compile::current();
853
854 C->log_inline_id(this);
855
856 C->add_string_late_inline(this);
857
858 JVMState* new_jvms = DirectCallGenerator::generate(jvms);
859 return new_jvms;
860 }
1049 // Inline failed, so make a direct call.
1050 assert(_if_hit->is_inline(), "must have been a failed inline");
1051 CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method());
1052 new_jvms = cg->generate(kit.sync_jvms());
1053 }
1054 kit.add_exception_states_from(new_jvms);
1055 kit.set_jvms(new_jvms);
1056
1057 // Need to merge slow and fast?
1058 if (slow_map == nullptr) {
1059 // The fast path is the only path remaining.
1060 return kit.transfer_exceptions_into_jvms();
1061 }
1062
1063 if (kit.stopped()) {
1064 // Inlined method threw an exception, so it's just the slow path after all.
1065 kit.set_jvms(slow_jvms);
1066 return kit.transfer_exceptions_into_jvms();
1067 }
1068
1069 // Allocate inline types if they are merged with objects (similar to Parse::merge_common())
1070 uint tos = kit.jvms()->stkoff() + kit.sp();
1071 uint limit = slow_map->req();
1072 for (uint i = TypeFunc::Parms; i < limit; i++) {
1073 Node* m = kit.map()->in(i);
1074 Node* n = slow_map->in(i);
1075 const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
1076 // TODO 8284443 still needed?
1077 if (m->is_InlineType() && !t->is_inlinetypeptr()) {
1078 // Allocate inline type in fast path
1079 m = m->as_InlineType()->buffer(&kit);
1080 kit.map()->set_req(i, m);
1081 }
1082 if (n->is_InlineType() && !t->is_inlinetypeptr()) {
1083 // Allocate inline type in slow path
1084 PreserveJVMState pjvms(&kit);
1085 kit.set_map(slow_map);
1086 n = n->as_InlineType()->buffer(&kit);
1087 kit.map()->set_req(i, n);
1088 slow_map = kit.stop();
1089 }
1090 }
1091
1092 // There are 2 branches and the replaced nodes are only valid on
1093 // one: restore the replaced nodes to what they were before the
1094 // branch.
1095 kit.map()->set_replaced_nodes(replaced_nodes);
1096
1097 // Finish the diamond.
1098 kit.C->set_has_split_ifs(true); // Has chance for split-if optimization
1099 RegionNode* region = new RegionNode(3);
1100 region->init_req(1, kit.control());
1101 region->init_req(2, slow_map->control());
1102 kit.set_control(gvn.transform(region));
1103 Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO);
1104 iophi->set_req(2, slow_map->i_o());
1105 kit.set_i_o(gvn.transform(iophi));
1106 // Merge memory
1107 kit.merge_memory(slow_map->merged_memory(), region, 2);
1108 // Transform new memory Phis.
1109 for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) {
1110 Node* phi = mms.memory();
1111 if (phi->is_Phi() && phi->in(0) == region) {
1112 mms.set_memory(gvn.transform(phi));
1113 }
1114 }
1115 for (uint i = TypeFunc::Parms; i < limit; i++) {
1116 // Skip unused stack slots; fast forward to monoff();
1117 if (i == tos) {
1118 i = kit.jvms()->monoff();
1119 if( i >= limit ) break;
1120 }
1121 Node* m = kit.map()->in(i);
1122 Node* n = slow_map->in(i);
1123 if (m != n) {
1124 const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
1125 Node* phi = PhiNode::make(region, m, t);
1126 phi->set_req(2, n);
1127 kit.map()->set_req(i, gvn.transform(phi));
1128 }
1129 }
1130 return kit.transfer_exceptions_into_jvms();
1131 }
1132
1133
1134 CallGenerator* CallGenerator::for_method_handle_call(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline) {
1135 assert(callee->is_method_handle_intrinsic(), "for_method_handle_call mismatch");
1136 bool input_not_const;
1137 CallGenerator* cg = CallGenerator::for_method_handle_inline(jvms, caller, callee, allow_inline, input_not_const);
1138 Compile* C = Compile::current();
1139 bool should_delay = C->should_delay_inlining();
1140 if (cg != nullptr) {
1141 if (should_delay && IncrementalInlineMH) {
1142 return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
1143 } else {
1144 return cg;
1145 }
1146 }
1147 int bci = jvms->bci();
1148 ciCallProfile profile = caller->call_profile_at_bci(bci);
1149 int call_site_count = caller->scale_count(profile.count());
1150
1151 if (IncrementalInlineMH && (AlwaysIncrementalInline ||
1152 (call_site_count > 0 && (should_delay || input_not_const || !C->inlining_incrementally() || C->over_inlining_cutoff())))) {
1153 return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
1154 } else {
1155 // Out-of-line call.
1156 return CallGenerator::for_direct_call(callee);
1157 }
1158 }
1159
1160
1161 CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline, bool& input_not_const) {
1162 GraphKit kit(jvms);
1163 PhaseGVN& gvn = kit.gvn();
1164 Compile* C = kit.C;
1165 vmIntrinsics::ID iid = callee->intrinsic_id();
1166 input_not_const = true;
1167 if (StressMethodHandleLinkerInlining) {
1168 allow_inline = false;
1169 }
1170 switch (iid) {
1171 case vmIntrinsics::_invokeBasic:
1172 {
1173 // Get MethodHandle receiver:
1174 Node* receiver = kit.argument(0);
1175 if (receiver->Opcode() == Op_ConP) {
1176 input_not_const = false;
1177 const TypeOopPtr* recv_toop = receiver->bottom_type()->isa_oopptr();
1178 if (recv_toop != nullptr) {
1179 ciMethod* target = recv_toop->const_oop()->as_method_handle()->get_vmtarget();
1180 const int vtable_index = Method::invalid_vtable_index;
1188 false /* call_does_dispatch */,
1189 jvms,
1190 allow_inline,
1191 PROB_ALWAYS);
1192 return cg;
1193 } else {
1194 assert(receiver->bottom_type() == TypePtr::NULL_PTR, "not a null: %s",
1195 Type::str(receiver->bottom_type()));
1196 print_inlining_failure(C, callee, jvms, "receiver is always null");
1197 }
1198 } else {
1199 print_inlining_failure(C, callee, jvms, "receiver not constant");
1200 }
1201 } break;
1202
1203 case vmIntrinsics::_linkToVirtual:
1204 case vmIntrinsics::_linkToStatic:
1205 case vmIntrinsics::_linkToSpecial:
1206 case vmIntrinsics::_linkToInterface:
1207 {
1208 int nargs = callee->arg_size();
1209 // Get MemberName argument:
1210 Node* member_name = kit.argument(nargs - 1);
1211 if (member_name->Opcode() == Op_ConP) {
1212 input_not_const = false;
1213 const TypeOopPtr* oop_ptr = member_name->bottom_type()->is_oopptr();
1214 ciMethod* target = oop_ptr->const_oop()->as_member_name()->get_vmtarget();
1215
1216 if (!ciMethod::is_consistent_info(callee, target)) {
1217 print_inlining_failure(C, callee, jvms, "signatures mismatch");
1218 return nullptr;
1219 }
1220
1221 // In lambda forms we erase signature types to avoid resolving issues
1222 // involving class loaders. When we optimize a method handle invoke
1223 // to a direct call we must cast the receiver and arguments to its
1224 // actual types.
1225 ciSignature* signature = target->signature();
1226 const int receiver_skip = target->is_static() ? 0 : 1;
1227 // Cast receiver to its type.
1228 if (!target->is_static()) {
1229 Node* recv = kit.argument(0);
1230 Node* casted_recv = kit.maybe_narrow_object_type(recv, signature->accessing_klass());
1260 ciKlass* speculative_receiver_type = nullptr;
1261 if (is_virtual_or_interface) {
1262 ciInstanceKlass* klass = target->holder();
1263 Node* receiver_node = kit.argument(0);
1264 const TypeOopPtr* receiver_type = gvn.type(receiver_node)->isa_oopptr();
1265 // call_does_dispatch and vtable_index are out-parameters. They might be changed.
1266 // optimize_virtual_call() takes 2 different holder
1267 // arguments for a corner case that doesn't apply here (see
1268 // Parse::do_call())
1269 target = C->optimize_virtual_call(caller, klass, klass,
1270 target, receiver_type, is_virtual,
1271 call_does_dispatch, vtable_index, // out-parameters
1272 false /* check_access */);
1273 // We lack profiling at this call but type speculation may
1274 // provide us with a type
1275 speculative_receiver_type = (receiver_type != nullptr) ? receiver_type->speculative_type() : nullptr;
1276 }
1277 CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms,
1278 allow_inline,
1279 PROB_ALWAYS,
1280 speculative_receiver_type,
1281 true);
1282 return cg;
1283 } else {
1284 print_inlining_failure(C, callee, jvms, "member_name not constant");
1285 }
1286 } break;
1287
1288 case vmIntrinsics::_linkToNative:
1289 print_inlining_failure(C, callee, jvms, "native call");
1290 break;
1291
1292 default:
1293 fatal("unexpected intrinsic %d: %s", vmIntrinsics::as_int(iid), vmIntrinsics::name_at(iid));
1294 break;
1295 }
1296 return nullptr;
1297 }
1298
1299 //------------------------PredicatedIntrinsicGenerator------------------------------
1300 // Internal class which handles all predicated Intrinsic calls.
1301 class PredicatedIntrinsicGenerator : public CallGenerator {
1333 // do_intrinsic(0)
1334 // else
1335 // if (predicate(1))
1336 // do_intrinsic(1)
1337 // ...
1338 // else
1339 // do_java_comp
1340
1341 GraphKit kit(jvms);
1342 PhaseGVN& gvn = kit.gvn();
1343
1344 CompileLog* log = kit.C->log();
1345 if (log != nullptr) {
1346 log->elem("predicated_intrinsic bci='%d' method='%d'",
1347 jvms->bci(), log->identify(method()));
1348 }
1349
1350 if (!method()->is_static()) {
1351 // We need an explicit receiver null_check before checking its type in predicate.
1352 // We share a map with the caller, so his JVMS gets adjusted.
1353 kit.null_check_receiver_before_call(method());
1354 if (kit.stopped()) {
1355 return kit.transfer_exceptions_into_jvms();
1356 }
1357 }
1358
1359 int n_predicates = _intrinsic->predicates_count();
1360 assert(n_predicates > 0, "sanity");
1361
1362 JVMState** result_jvms = NEW_RESOURCE_ARRAY(JVMState*, (n_predicates+1));
1363
1364 // Region for normal compilation code if intrinsic failed.
1365 Node* slow_region = new RegionNode(1);
1366
1367 int results = 0;
1368 for (int predicate = 0; (predicate < n_predicates) && !kit.stopped(); predicate++) {
1369 #ifdef ASSERT
1370 JVMState* old_jvms = kit.jvms();
1371 SafePointNode* old_map = kit.map();
1372 Node* old_io = old_map->i_o();
1373 Node* old_mem = old_map->memory();
|