18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/bcEscapeAnalyzer.hpp"
27 #include "ci/ciCallSite.hpp"
28 #include "ci/ciObjArray.hpp"
29 #include "ci/ciMemberName.hpp"
30 #include "ci/ciMethodHandle.hpp"
31 #include "classfile/javaClasses.hpp"
32 #include "compiler/compileLog.hpp"
33 #include "opto/addnode.hpp"
34 #include "opto/callGenerator.hpp"
35 #include "opto/callnode.hpp"
36 #include "opto/castnode.hpp"
37 #include "opto/cfgnode.hpp"
38 #include "opto/parse.hpp"
39 #include "opto/rootnode.hpp"
40 #include "opto/runtime.hpp"
41 #include "opto/subnode.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "ci/ciNativeEntryPoint.hpp"
44 #include "utilities/debug.hpp"
45
46 // Utility function.
47 const TypeFunc* CallGenerator::tf() const {
48 return TypeFunc::make(method());
49 }
50
51 bool CallGenerator::is_inlined_method_handle_intrinsic(JVMState* jvms, ciMethod* m) {
52 return is_inlined_method_handle_intrinsic(jvms->method(), jvms->bci(), m);
53 }
54
55 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* caller, int bci, ciMethod* m) {
56 ciMethod* symbolic_info = caller->get_method_at_bci(bci);
57 return is_inlined_method_handle_intrinsic(symbolic_info, m);
101 GraphKit& exits = parser.exits();
102
103 if (C->failing()) {
104 while (exits.pop_exception_state() != NULL) ;
105 return NULL;
106 }
107
108 assert(exits.jvms()->same_calls_as(jvms), "sanity");
109
110 // Simply return the exit state of the parser,
111 // augmented by any exceptional states.
112 return exits.transfer_exceptions_into_jvms();
113 }
114
115 //---------------------------DirectCallGenerator------------------------------
116 // Internal class which handles all out-of-line calls w/o receiver type checks.
117 class DirectCallGenerator : public CallGenerator {
118 private:
119 CallStaticJavaNode* _call_node;
120 // Force separate memory and I/O projections for the exceptional
121 // paths to facilitate late inlinig.
122 bool _separate_io_proj;
123
124 protected:
125 void set_call_node(CallStaticJavaNode* call) { _call_node = call; }
126
127 public:
128 DirectCallGenerator(ciMethod* method, bool separate_io_proj)
129 : CallGenerator(method),
130 _separate_io_proj(separate_io_proj)
131 {
132 }
133 virtual JVMState* generate(JVMState* jvms);
134
135 virtual CallNode* call_node() const { return _call_node; }
136 virtual CallGenerator* with_call_node(CallNode* call) {
137 DirectCallGenerator* dcg = new DirectCallGenerator(method(), _separate_io_proj);
138 dcg->set_call_node(call->as_CallStaticJava());
139 return dcg;
140 }
141 };
142
143 JVMState* DirectCallGenerator::generate(JVMState* jvms) {
144 GraphKit kit(jvms);
145 kit.C->print_inlining_update(this);
146 bool is_static = method()->is_static();
147 address target = is_static ? SharedRuntime::get_resolve_static_call_stub()
148 : SharedRuntime::get_resolve_opt_virtual_call_stub();
149
150 if (kit.C->log() != NULL) {
151 kit.C->log()->elem("direct_call bci='%d'", jvms->bci());
152 }
153
154 CallStaticJavaNode* call = new CallStaticJavaNode(kit.C, tf(), target, method());
155 if (is_inlined_method_handle_intrinsic(jvms, method())) {
156 // To be able to issue a direct call and skip a call to MH.linkTo*/invokeBasic adapter,
157 // additional information about the method being invoked should be attached
158 // to the call site to make resolution logic work
159 // (see SharedRuntime::resolve_static_call_C).
160 call->set_override_symbolic_info(true);
161 }
162 _call_node = call; // Save the call node in case we need it later
163 if (!is_static) {
164 // Make an explicit receiver null_check as part of this call.
165 // Since we share a map with the caller, his JVMS gets adjusted.
166 kit.null_check_receiver_before_call(method());
167 if (kit.stopped()) {
168 // And dump it back to the caller, decorated with any exceptions:
169 return kit.transfer_exceptions_into_jvms();
170 }
171 // Mark the call node as virtual, sort of:
172 call->set_optimized_virtual(true);
173 if (method()->is_method_handle_intrinsic() ||
174 method()->is_compiled_lambda_form()) {
175 call->set_method_handle_invoke(true);
176 }
177 }
178 kit.set_arguments_for_java_call(call);
179 kit.set_edges_for_java_call(call, false, _separate_io_proj);
180 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
181 kit.push_node(method()->return_type()->basic_type(), ret);
182 return kit.transfer_exceptions_into_jvms();
183 }
184
185 //--------------------------VirtualCallGenerator------------------------------
186 // Internal class which handles all out-of-line calls checking receiver type.
187 class VirtualCallGenerator : public CallGenerator {
188 private:
189 int _vtable_index;
190 bool _separate_io_proj;
191 CallDynamicJavaNode* _call_node;
192
193 protected:
194 void set_call_node(CallDynamicJavaNode* call) { _call_node = call; }
195
196 public:
197 VirtualCallGenerator(ciMethod* method, int vtable_index, bool separate_io_proj)
198 : CallGenerator(method), _vtable_index(vtable_index), _separate_io_proj(separate_io_proj), _call_node(NULL)
199 {
200 assert(vtable_index == Method::invalid_vtable_index ||
201 vtable_index >= 0, "either invalid or usable");
202 }
203 virtual bool is_virtual() const { return true; }
204 virtual JVMState* generate(JVMState* jvms);
205
206 virtual CallNode* call_node() const { return _call_node; }
207 int vtable_index() const { return _vtable_index; }
208
209 virtual CallGenerator* with_call_node(CallNode* call) {
210 VirtualCallGenerator* cg = new VirtualCallGenerator(method(), _vtable_index, _separate_io_proj);
211 cg->set_call_node(call->as_CallDynamicJava());
212 return cg;
213 }
214 };
215
216 JVMState* VirtualCallGenerator::generate(JVMState* jvms) {
217 GraphKit kit(jvms);
218 Node* receiver = kit.argument(0);
219
220 kit.C->print_inlining_update(this);
221
222 if (kit.C->log() != NULL) {
223 kit.C->log()->elem("virtual_call bci='%d'", jvms->bci());
224 }
225
226 // If the receiver is a constant null, do not torture the system
227 // by attempting to call through it. The compile will proceed
228 // correctly, but may bail out in final_graph_reshaping, because
229 // the call instruction will have a seemingly deficient out-count.
230 // (The bailout says something misleading about an "infinite loop".)
231 if (kit.gvn().type(receiver)->higher_equal(TypePtr::NULL_PTR)) {
232 assert(Bytecodes::is_invoke(kit.java_bc()), "%d: %s", kit.java_bc(), Bytecodes::name(kit.java_bc()));
233 ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci());
234 int arg_size = declared_method->signature()->arg_size_for_bc(kit.java_bc());
235 kit.inc_sp(arg_size); // restore arguments
236 kit.uncommon_trap(Deoptimization::Reason_null_check,
237 Deoptimization::Action_none,
238 NULL, "null receiver");
239 return kit.transfer_exceptions_into_jvms();
240 }
241
242 // Ideally we would unconditionally do a null check here and let it
243 // be converted to an implicit check based on profile information.
244 // However currently the conversion to implicit null checks in
245 // Block::implicit_null_check() only looks for loads and stores, not calls.
246 ciMethod *caller = kit.method();
247 ciMethodData *caller_md = (caller == NULL) ? NULL : caller->method_data();
248 if (!UseInlineCaches || !ImplicitNullChecks || !os::zero_page_read_protected() ||
249 ((ImplicitNullCheckThreshold > 0) && caller_md &&
250 (caller_md->trap_count(Deoptimization::Reason_null_check)
251 >= (uint)ImplicitNullCheckThreshold))) {
259 }
260
261 assert(!method()->is_static(), "virtual call must not be to static");
262 assert(!method()->is_final(), "virtual call should not be to final");
263 assert(!method()->is_private(), "virtual call should not be to private");
264 assert(_vtable_index == Method::invalid_vtable_index || !UseInlineCaches,
265 "no vtable calls if +UseInlineCaches ");
266 address target = SharedRuntime::get_resolve_virtual_call_stub();
267 // Normal inline cache used for call
268 CallDynamicJavaNode* call = new CallDynamicJavaNode(tf(), target, method(), _vtable_index);
269 if (is_inlined_method_handle_intrinsic(jvms, method())) {
270 // To be able to issue a direct call (optimized virtual or virtual)
271 // and skip a call to MH.linkTo*/invokeBasic adapter, additional information
272 // about the method being invoked should be attached to the call site to
273 // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C).
274 call->set_override_symbolic_info(true);
275 }
276 _call_node = call; // Save the call node in case we need it later
277
278 kit.set_arguments_for_java_call(call);
279 kit.set_edges_for_java_call(call, false /*must_throw*/, _separate_io_proj);
280 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
281 kit.push_node(method()->return_type()->basic_type(), ret);
282
283 // Represent the effect of an implicit receiver null_check
284 // as part of this call. Since we share a map with the caller,
285 // his JVMS gets adjusted.
286 kit.cast_not_null(receiver);
287 return kit.transfer_exceptions_into_jvms();
288 }
289
290 CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) {
291 if (InlineTree::check_can_parse(m) != NULL) return NULL;
292 return new ParseGenerator(m, expected_uses);
293 }
294
295 // As a special case, the JVMS passed to this CallGenerator is
296 // for the method execution already in progress, not just the JVMS
297 // of the caller. Thus, this CallGenerator cannot be mixed with others!
298 CallGenerator* CallGenerator::for_osr(ciMethod* m, int osr_bci) {
352 return DirectCallGenerator::generate(jvms);
353 }
354
355 virtual void print_inlining_late(const char* msg) {
356 CallNode* call = call_node();
357 Compile* C = Compile::current();
358 C->print_inlining_assert_ready();
359 C->print_inlining(method(), call->jvms()->depth()-1, call->jvms()->bci(), msg);
360 C->print_inlining_move_to(this);
361 C->print_inlining_update_delayed(this);
362 }
363
364 virtual void set_unique_id(jlong id) {
365 _unique_id = id;
366 }
367
368 virtual jlong unique_id() const {
369 return _unique_id;
370 }
371
372 virtual CallGenerator* with_call_node(CallNode* call) {
373 LateInlineCallGenerator* cg = new LateInlineCallGenerator(method(), _inline_cg, _is_pure_call);
374 cg->set_call_node(call->as_CallStaticJava());
375 return cg;
376 }
377 };
378
379 CallGenerator* CallGenerator::for_late_inline(ciMethod* method, CallGenerator* inline_cg) {
380 return new LateInlineCallGenerator(method, inline_cg);
381 }
382
383 class LateInlineMHCallGenerator : public LateInlineCallGenerator {
384 ciMethod* _caller;
385 bool _input_not_const;
386
387 virtual bool do_late_inline_check(Compile* C, JVMState* jvms);
388
389 public:
390 LateInlineMHCallGenerator(ciMethod* caller, ciMethod* callee, bool input_not_const) :
391 LateInlineCallGenerator(callee, NULL), _caller(caller), _input_not_const(input_not_const) {}
406 C->add_late_inline(this);
407 }
408 return new_jvms;
409 }
410
411 virtual CallGenerator* with_call_node(CallNode* call) {
412 LateInlineMHCallGenerator* cg = new LateInlineMHCallGenerator(_caller, method(), _input_not_const);
413 cg->set_call_node(call->as_CallStaticJava());
414 return cg;
415 }
416 };
417
418 bool LateInlineMHCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) {
419 // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call.
420 bool allow_inline = C->inlining_incrementally();
421 bool input_not_const = true;
422 CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), allow_inline, input_not_const);
423 assert(!input_not_const, "sanity"); // shouldn't have been scheduled for inlining in the first place
424
425 if (cg != NULL) {
426 assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline, "we're doing late inlining");
427 _inline_cg = cg;
428 C->dec_number_of_mh_late_inlines();
429 return true;
430 } else {
431 // Method handle call which has a constant appendix argument should be either inlined or replaced with a direct call
432 // unless there's a signature mismatch between caller and callee. If the failure occurs, there's not much to be improved later,
433 // so don't reinstall the generator to avoid pushing the generator between IGVN and incremental inlining indefinitely.
434 return false;
435 }
436 }
437
438 CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const) {
439 assert(IncrementalInlineMH, "required");
440 Compile::current()->inc_number_of_mh_late_inlines();
441 CallGenerator* cg = new LateInlineMHCallGenerator(caller, callee, input_not_const);
442 return cg;
443 }
444
445 // Allow inlining decisions to be delayed
616
617 #ifndef PRODUCT
618 if (PrintEliminateAllocations) {
619 tty->print("++++ Eliminated: %d ", call->_idx);
620 call->as_CallStaticJava()->method()->print_short_name(tty);
621 tty->cr();
622 }
623 #endif
624 }
625
626 void CallGenerator::do_late_inline_helper() {
627 assert(is_late_inline(), "only late inline allowed");
628
629 // Can't inline it
630 CallNode* call = call_node();
631 if (call == NULL || call->outcnt() == 0 ||
632 call->in(0) == NULL || call->in(0)->is_top()) {
633 return;
634 }
635
636 const TypeTuple *r = call->tf()->domain();
637 for (int i1 = 0; i1 < method()->arg_size(); i1++) {
638 if (call->in(TypeFunc::Parms + i1)->is_top() && r->field_at(TypeFunc::Parms + i1) != Type::HALF) {
639 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
640 return;
641 }
642 }
643
644 if (call->in(TypeFunc::Memory)->is_top()) {
645 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
646 return;
647 }
648 if (call->in(TypeFunc::Memory)->is_MergeMem()) {
649 MergeMemNode* merge_mem = call->in(TypeFunc::Memory)->as_MergeMem();
650 if (merge_mem->base_memory() == merge_mem->empty_memory()) {
651 return; // dead path
652 }
653 }
654
655 // check for unreachable loop
656 CallProjections callprojs;
657 call->extract_projections(&callprojs, true);
658 if ((callprojs.fallthrough_catchproj == call->in(0)) ||
659 (callprojs.catchall_catchproj == call->in(0)) ||
660 (callprojs.fallthrough_memproj == call->in(TypeFunc::Memory)) ||
661 (callprojs.catchall_memproj == call->in(TypeFunc::Memory)) ||
662 (callprojs.fallthrough_ioproj == call->in(TypeFunc::I_O)) ||
663 (callprojs.catchall_ioproj == call->in(TypeFunc::I_O)) ||
664 (callprojs.resproj != NULL && call->find_edge(callprojs.resproj) != -1) ||
665 (callprojs.exobj != NULL && call->find_edge(callprojs.exobj) != -1)) {
666 return;
667 }
668
669 Compile* C = Compile::current();
670 // Remove inlined methods from Compiler's lists.
671 if (call->is_macro()) {
672 C->remove_macro_node(call);
673 }
674
675 bool result_not_used = false;
676
677 if (is_pure_call()) {
678 if (is_boxing_late_inline() && callprojs.resproj != nullptr) {
679 // replace box node to scalar node only in case it is directly referenced by debug info
680 assert(call->as_CallStaticJava()->is_boxing_method(), "sanity");
681 if (!has_non_debug_usages(callprojs.resproj) && is_box_cache_valid(call)) {
682 scalarize_debug_usages(call, callprojs.resproj);
683 }
684 }
685
686 // The call is marked as pure (no important side effects), but result isn't used.
687 // It's safe to remove the call.
688 result_not_used = (callprojs.resproj == NULL || callprojs.resproj->outcnt() == 0);
689 }
690
691 if (result_not_used) {
692 GraphKit kit(call->jvms());
693 kit.replace_call(call, C->top(), true);
694 } else {
695 // Make a clone of the JVMState that appropriate to use for driving a parse
696 JVMState* old_jvms = call->jvms();
697 JVMState* jvms = old_jvms->clone_shallow(C);
698 uint size = call->req();
699 SafePointNode* map = new SafePointNode(size, jvms);
700 for (uint i1 = 0; i1 < size; i1++) {
701 map->init_req(i1, call->in(i1));
702 }
703
704 // Make sure the state is a MergeMem for parsing.
705 if (!map->in(TypeFunc::Memory)->is_MergeMem()) {
706 Node* mem = MergeMemNode::make(map->in(TypeFunc::Memory));
707 C->initial_gvn()->set_type_bottom(mem);
708 map->set_req(TypeFunc::Memory, mem);
709 }
710
711 uint nargs = method()->arg_size();
712 // blow away old call arguments
713 Node* top = C->top();
714 for (uint i1 = 0; i1 < nargs; i1++) {
715 map->set_req(TypeFunc::Parms + i1, top);
716 }
717 jvms->set_map(map);
718
719 // Make enough space in the expression stack to transfer
720 // the incoming arguments and return value.
721 map->ensure_stack(jvms, jvms->method()->max_stack());
722 for (uint i1 = 0; i1 < nargs; i1++) {
723 map->set_argument(jvms, i1, call->in(TypeFunc::Parms + i1));
724 }
725
726 C->print_inlining_assert_ready();
727
728 C->print_inlining_move_to(this);
729
730 C->log_late_inline(this);
731
732 // JVMState is ready, so time to perform some checks and prepare for inlining attempt.
733 if (!do_late_inline_check(C, jvms)) {
734 map->disconnect_inputs(C);
735 C->print_inlining_update_delayed(this);
736 return;
737 }
738
739 // Setup default node notes to be picked up by the inlining
740 Node_Notes* old_nn = C->node_notes_at(call->_idx);
741 if (old_nn != NULL) {
742 Node_Notes* entry_nn = old_nn->clone(C);
743 entry_nn->set_jvms(jvms);
744 C->set_default_node_notes(entry_nn);
745 }
746
747 // Now perform the inlining using the synthesized JVMState
748 JVMState* new_jvms = inline_cg()->generate(jvms);
749 if (new_jvms == NULL) return; // no change
750 if (C->failing()) return;
751
752 // Capture any exceptional control flow
753 GraphKit kit(new_jvms);
754
755 // Find the result object
756 Node* result = C->top();
757 int result_size = method()->return_type()->size();
758 if (result_size != 0 && !kit.stopped()) {
759 result = (result_size == 1) ? kit.pop() : kit.pop_pair();
760 }
761
762 if (inline_cg()->is_inline()) {
763 C->set_has_loops(C->has_loops() || inline_cg()->method()->has_loops());
764 C->env()->notice_inlined_method(inline_cg()->method());
765 }
766 C->set_inlining_progress(true);
767 C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup
768 kit.replace_call(call, result, true);
769 }
770 }
771
772 class LateInlineStringCallGenerator : public LateInlineCallGenerator {
773
774 public:
775 LateInlineStringCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
776 LateInlineCallGenerator(method, inline_cg) {}
777
778 virtual JVMState* generate(JVMState* jvms) {
779 Compile *C = Compile::current();
780
781 C->log_inline_id(this);
782
783 C->add_string_late_inline(this);
784
785 JVMState* new_jvms = DirectCallGenerator::generate(jvms);
786 return new_jvms;
787 }
972 // Inline failed, so make a direct call.
973 assert(_if_hit->is_inline(), "must have been a failed inline");
974 CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method());
975 new_jvms = cg->generate(kit.sync_jvms());
976 }
977 kit.add_exception_states_from(new_jvms);
978 kit.set_jvms(new_jvms);
979
980 // Need to merge slow and fast?
981 if (slow_map == NULL) {
982 // The fast path is the only path remaining.
983 return kit.transfer_exceptions_into_jvms();
984 }
985
986 if (kit.stopped()) {
987 // Inlined method threw an exception, so it's just the slow path after all.
988 kit.set_jvms(slow_jvms);
989 return kit.transfer_exceptions_into_jvms();
990 }
991
992 // There are 2 branches and the replaced nodes are only valid on
993 // one: restore the replaced nodes to what they were before the
994 // branch.
995 kit.map()->set_replaced_nodes(replaced_nodes);
996
997 // Finish the diamond.
998 kit.C->set_has_split_ifs(true); // Has chance for split-if optimization
999 RegionNode* region = new RegionNode(3);
1000 region->init_req(1, kit.control());
1001 region->init_req(2, slow_map->control());
1002 kit.set_control(gvn.transform(region));
1003 Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO);
1004 iophi->set_req(2, slow_map->i_o());
1005 kit.set_i_o(gvn.transform(iophi));
1006 // Merge memory
1007 kit.merge_memory(slow_map->merged_memory(), region, 2);
1008 // Transform new memory Phis.
1009 for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) {
1010 Node* phi = mms.memory();
1011 if (phi->is_Phi() && phi->in(0) == region) {
1012 mms.set_memory(gvn.transform(phi));
1013 }
1014 }
1015 uint tos = kit.jvms()->stkoff() + kit.sp();
1016 uint limit = slow_map->req();
1017 for (uint i = TypeFunc::Parms; i < limit; i++) {
1018 // Skip unused stack slots; fast forward to monoff();
1019 if (i == tos) {
1020 i = kit.jvms()->monoff();
1021 if( i >= limit ) break;
1022 }
1023 Node* m = kit.map()->in(i);
1024 Node* n = slow_map->in(i);
1025 if (m != n) {
1026 const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
1027 Node* phi = PhiNode::make(region, m, t);
1028 phi->set_req(2, n);
1029 kit.map()->set_req(i, gvn.transform(phi));
1030 }
1031 }
1032 return kit.transfer_exceptions_into_jvms();
1033 }
1034
1035
1036 CallGenerator* CallGenerator::for_method_handle_call(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline) {
1037 assert(callee->is_method_handle_intrinsic(), "for_method_handle_call mismatch");
1038 bool input_not_const;
1039 CallGenerator* cg = CallGenerator::for_method_handle_inline(jvms, caller, callee, allow_inline, input_not_const);
1040 Compile* C = Compile::current();
1041 if (cg != NULL) {
1042 if (AlwaysIncrementalInline) {
1043 return CallGenerator::for_late_inline(callee, cg);
1044 } else {
1045 return cg;
1046 }
1047 }
1048 int bci = jvms->bci();
1049 ciCallProfile profile = caller->call_profile_at_bci(bci);
1050 int call_site_count = caller->scale_count(profile.count());
1051
1052 if (IncrementalInlineMH && call_site_count > 0 &&
1053 (input_not_const || !C->inlining_incrementally() || C->over_inlining_cutoff())) {
1054 return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
1055 } else {
1056 // Out-of-line call.
1057 return CallGenerator::for_direct_call(callee);
1058 }
1059 }
1060
1061 class NativeCallGenerator : public CallGenerator {
1062 private:
1063 address _call_addr;
1064 ciNativeEntryPoint* _nep;
1065 public:
1066 NativeCallGenerator(ciMethod* m, address call_addr, ciNativeEntryPoint* nep)
1067 : CallGenerator(m), _call_addr(call_addr), _nep(nep) {}
1068
1069 virtual JVMState* generate(JVMState* jvms);
1070 };
1071
1072 JVMState* NativeCallGenerator::generate(JVMState* jvms) {
1073 GraphKit kit(jvms);
1074
1075 Node* call = kit.make_native_call(_call_addr, tf(), method()->arg_size(), _nep); // -fallback, - nep
1076 if (call == NULL) return NULL;
1077
1078 kit.C->print_inlining_update(this);
1079 if (kit.C->log() != NULL) {
1080 kit.C->log()->elem("l2n_intrinsification_success bci='%d' entry_point='" INTPTR_FORMAT "'", jvms->bci(), p2i(_call_addr));
1095 switch (iid) {
1096 case vmIntrinsics::_invokeBasic:
1097 {
1098 // Get MethodHandle receiver:
1099 Node* receiver = kit.argument(0);
1100 if (receiver->Opcode() == Op_ConP) {
1101 input_not_const = false;
1102 const TypeOopPtr* oop_ptr = receiver->bottom_type()->is_oopptr();
1103 ciMethod* target = oop_ptr->const_oop()->as_method_handle()->get_vmtarget();
1104 const int vtable_index = Method::invalid_vtable_index;
1105
1106 if (!ciMethod::is_consistent_info(callee, target)) {
1107 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1108 "signatures mismatch");
1109 return NULL;
1110 }
1111
1112 CallGenerator* cg = C->call_generator(target, vtable_index,
1113 false /* call_does_dispatch */,
1114 jvms,
1115 allow_inline,
1116 PROB_ALWAYS);
1117 return cg;
1118 } else {
1119 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1120 "receiver not constant");
1121 }
1122 }
1123 break;
1124
1125 case vmIntrinsics::_linkToVirtual:
1126 case vmIntrinsics::_linkToStatic:
1127 case vmIntrinsics::_linkToSpecial:
1128 case vmIntrinsics::_linkToInterface:
1129 {
1130 // Get MemberName argument:
1131 Node* member_name = kit.argument(callee->arg_size() - 1);
1132 if (member_name->Opcode() == Op_ConP) {
1133 input_not_const = false;
1134 const TypeOopPtr* oop_ptr = member_name->bottom_type()->is_oopptr();
1135 ciMethod* target = oop_ptr->const_oop()->as_member_name()->get_vmtarget();
1136
1137 if (!ciMethod::is_consistent_info(callee, target)) {
1138 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1139 "signatures mismatch");
1140 return NULL;
1141 }
1142
1143 // In lambda forms we erase signature types to avoid resolving issues
1144 // involving class loaders. When we optimize a method handle invoke
1145 // to a direct call we must cast the receiver and arguments to its
1146 // actual types.
1147 ciSignature* signature = target->signature();
1148 const int receiver_skip = target->is_static() ? 0 : 1;
1149 // Cast receiver to its type.
1150 if (!target->is_static()) {
1151 Node* arg = kit.argument(0);
1152 const TypeOopPtr* arg_type = arg->bottom_type()->isa_oopptr();
1153 const Type* sig_type = TypeOopPtr::make_from_klass(signature->accessing_klass());
1154 if (arg_type != NULL && !arg_type->higher_equal(sig_type)) {
1155 const Type* recv_type = arg_type->filter_speculative(sig_type); // keep speculative part
1156 Node* cast_obj = gvn.transform(new CheckCastPPNode(kit.control(), arg, recv_type));
1157 kit.set_argument(0, cast_obj);
1158 }
1159 }
1160 // Cast reference arguments to its type.
1161 for (int i = 0, j = 0; i < signature->count(); i++) {
1162 ciType* t = signature->type_at(i);
1163 if (t->is_klass()) {
1164 Node* arg = kit.argument(receiver_skip + j);
1165 const TypeOopPtr* arg_type = arg->bottom_type()->isa_oopptr();
1166 const Type* sig_type = TypeOopPtr::make_from_klass(t->as_klass());
1167 if (arg_type != NULL && !arg_type->higher_equal(sig_type)) {
1168 const Type* narrowed_arg_type = arg_type->filter_speculative(sig_type); // keep speculative part
1169 Node* cast_obj = gvn.transform(new CheckCastPPNode(kit.control(), arg, narrowed_arg_type));
1170 kit.set_argument(receiver_skip + j, cast_obj);
1171 }
1172 }
1173 j += t->size(); // long and double take two slots
1174 }
1175
1176 // Try to get the most accurate receiver type
1177 const bool is_virtual = (iid == vmIntrinsics::_linkToVirtual);
1178 const bool is_virtual_or_interface = (is_virtual || iid == vmIntrinsics::_linkToInterface);
1179 int vtable_index = Method::invalid_vtable_index;
1180 bool call_does_dispatch = false;
1181
1182 ciKlass* speculative_receiver_type = NULL;
1183 if (is_virtual_or_interface) {
1184 ciInstanceKlass* klass = target->holder();
1185 Node* receiver_node = kit.argument(0);
1186 const TypeOopPtr* receiver_type = gvn.type(receiver_node)->isa_oopptr();
1187 // call_does_dispatch and vtable_index are out-parameters. They might be changed.
1188 // optimize_virtual_call() takes 2 different holder
1189 // arguments for a corner case that doesn't apply here (see
1190 // Parse::do_call())
1191 target = C->optimize_virtual_call(caller, klass, klass,
1192 target, receiver_type, is_virtual,
1193 call_does_dispatch, vtable_index, // out-parameters
1194 false /* check_access */);
1195 // We lack profiling at this call but type speculation may
1196 // provide us with a type
1197 speculative_receiver_type = (receiver_type != NULL) ? receiver_type->speculative_type() : NULL;
1198 }
1199 CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms,
1200 allow_inline,
1201 PROB_ALWAYS,
1202 speculative_receiver_type);
1203 return cg;
1204 } else {
1205 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1206 "member_name not constant");
1207 }
1208 }
1209 break;
1210
1211 case vmIntrinsics::_linkToNative:
1212 {
1213 Node* addr_n = kit.argument(1); // target address
1214 Node* nep_n = kit.argument(callee->arg_size() - 1); // NativeEntryPoint
1215 // This check needs to be kept in sync with the one in CallStaticJavaNode::Ideal
1216 if (addr_n->Opcode() == Op_ConL && nep_n->Opcode() == Op_ConP) {
1217 input_not_const = false;
1218 const TypeLong* addr_t = addr_n->bottom_type()->is_long();
1219 const TypeOopPtr* nep_t = nep_n->bottom_type()->is_oopptr();
1220 address addr = (address) addr_t->get_con();
1221 ciNativeEntryPoint* nep = nep_t->const_oop()->as_native_entry_point();
1222 return new NativeCallGenerator(callee, addr, nep);
1272 // do_intrinsic(0)
1273 // else
1274 // if (predicate(1))
1275 // do_intrinsic(1)
1276 // ...
1277 // else
1278 // do_java_comp
1279
1280 GraphKit kit(jvms);
1281 PhaseGVN& gvn = kit.gvn();
1282
1283 CompileLog* log = kit.C->log();
1284 if (log != NULL) {
1285 log->elem("predicated_intrinsic bci='%d' method='%d'",
1286 jvms->bci(), log->identify(method()));
1287 }
1288
1289 if (!method()->is_static()) {
1290 // We need an explicit receiver null_check before checking its type in predicate.
1291 // We share a map with the caller, so his JVMS gets adjusted.
1292 Node* receiver = kit.null_check_receiver_before_call(method());
1293 if (kit.stopped()) {
1294 return kit.transfer_exceptions_into_jvms();
1295 }
1296 }
1297
1298 int n_predicates = _intrinsic->predicates_count();
1299 assert(n_predicates > 0, "sanity");
1300
1301 JVMState** result_jvms = NEW_RESOURCE_ARRAY(JVMState*, (n_predicates+1));
1302
1303 // Region for normal compilation code if intrinsic failed.
1304 Node* slow_region = new RegionNode(1);
1305
1306 int results = 0;
1307 for (int predicate = 0; (predicate < n_predicates) && !kit.stopped(); predicate++) {
1308 #ifdef ASSERT
1309 JVMState* old_jvms = kit.jvms();
1310 SafePointNode* old_map = kit.map();
1311 Node* old_io = old_map->i_o();
1312 Node* old_mem = old_map->memory();
|
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/bcEscapeAnalyzer.hpp"
27 #include "ci/ciCallSite.hpp"
28 #include "ci/ciObjArray.hpp"
29 #include "ci/ciMemberName.hpp"
30 #include "ci/ciMethodHandle.hpp"
31 #include "classfile/javaClasses.hpp"
32 #include "compiler/compileLog.hpp"
33 #include "opto/addnode.hpp"
34 #include "opto/callGenerator.hpp"
35 #include "opto/callnode.hpp"
36 #include "opto/castnode.hpp"
37 #include "opto/cfgnode.hpp"
38 #include "opto/inlinetypenode.hpp"
39 #include "opto/parse.hpp"
40 #include "opto/rootnode.hpp"
41 #include "opto/runtime.hpp"
42 #include "opto/subnode.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "ci/ciNativeEntryPoint.hpp"
45 #include "utilities/debug.hpp"
46
47 // Utility function.
48 const TypeFunc* CallGenerator::tf() const {
49 return TypeFunc::make(method());
50 }
51
52 bool CallGenerator::is_inlined_method_handle_intrinsic(JVMState* jvms, ciMethod* m) {
53 return is_inlined_method_handle_intrinsic(jvms->method(), jvms->bci(), m);
54 }
55
56 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* caller, int bci, ciMethod* m) {
57 ciMethod* symbolic_info = caller->get_method_at_bci(bci);
58 return is_inlined_method_handle_intrinsic(symbolic_info, m);
102 GraphKit& exits = parser.exits();
103
104 if (C->failing()) {
105 while (exits.pop_exception_state() != NULL) ;
106 return NULL;
107 }
108
109 assert(exits.jvms()->same_calls_as(jvms), "sanity");
110
111 // Simply return the exit state of the parser,
112 // augmented by any exceptional states.
113 return exits.transfer_exceptions_into_jvms();
114 }
115
116 //---------------------------DirectCallGenerator------------------------------
117 // Internal class which handles all out-of-line calls w/o receiver type checks.
118 class DirectCallGenerator : public CallGenerator {
119 private:
120 CallStaticJavaNode* _call_node;
121 // Force separate memory and I/O projections for the exceptional
122 // paths to facilitate late inlining.
123 bool _separate_io_proj;
124
125 protected:
126 void set_call_node(CallStaticJavaNode* call) { _call_node = call; }
127
128 public:
129 DirectCallGenerator(ciMethod* method, bool separate_io_proj)
130 : CallGenerator(method),
131 _call_node(NULL),
132 _separate_io_proj(separate_io_proj)
133 {
134 if (InlineTypeReturnedAsFields && method->is_method_handle_intrinsic()) {
135 // If that call has not been optimized by the time optimizations are over,
136 // we'll need to add a call to create an inline type instance from the klass
137 // returned by the call (see PhaseMacroExpand::expand_mh_intrinsic_return).
138 // Separating memory and I/O projections for exceptions is required to
139 // perform that graph transformation.
140 _separate_io_proj = true;
141 }
142 }
143 virtual JVMState* generate(JVMState* jvms);
144
145 virtual CallNode* call_node() const { return _call_node; }
146 virtual CallGenerator* with_call_node(CallNode* call) {
147 DirectCallGenerator* dcg = new DirectCallGenerator(method(), _separate_io_proj);
148 dcg->set_call_node(call->as_CallStaticJava());
149 return dcg;
150 }
151 };
152
153 JVMState* DirectCallGenerator::generate(JVMState* jvms) {
154 GraphKit kit(jvms);
155 kit.C->print_inlining_update(this);
156 PhaseGVN& gvn = kit.gvn();
157 bool is_static = method()->is_static();
158 address target = is_static ? SharedRuntime::get_resolve_static_call_stub()
159 : SharedRuntime::get_resolve_opt_virtual_call_stub();
160
161 if (kit.C->log() != NULL) {
162 kit.C->log()->elem("direct_call bci='%d'", jvms->bci());
163 }
164
165 CallStaticJavaNode* call = new CallStaticJavaNode(kit.C, tf(), target, method());
166 if (is_inlined_method_handle_intrinsic(jvms, method())) {
167 // To be able to issue a direct call and skip a call to MH.linkTo*/invokeBasic adapter,
168 // additional information about the method being invoked should be attached
169 // to the call site to make resolution logic work
170 // (see SharedRuntime::resolve_static_call_C).
171 call->set_override_symbolic_info(true);
172 }
173 _call_node = call; // Save the call node in case we need it later
174 if (!is_static) {
175 // Make an explicit receiver null_check as part of this call.
176 // Since we share a map with the caller, his JVMS gets adjusted.
177 kit.null_check_receiver_before_call(method());
178 if (kit.stopped()) {
179 // And dump it back to the caller, decorated with any exceptions:
180 return kit.transfer_exceptions_into_jvms();
181 }
182 // Mark the call node as virtual, sort of:
183 call->set_optimized_virtual(true);
184 if (method()->is_method_handle_intrinsic() ||
185 method()->is_compiled_lambda_form()) {
186 call->set_method_handle_invoke(true);
187 }
188 }
189 kit.set_arguments_for_java_call(call, is_late_inline());
190 if (kit.stopped()) {
191 return kit.transfer_exceptions_into_jvms();
192 }
193 kit.set_edges_for_java_call(call, false, _separate_io_proj);
194 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
195 kit.push_node(method()->return_type()->basic_type(), ret);
196 return kit.transfer_exceptions_into_jvms();
197 }
198
199 //--------------------------VirtualCallGenerator------------------------------
200 // Internal class which handles all out-of-line calls checking receiver type.
201 class VirtualCallGenerator : public CallGenerator {
202 private:
203 int _vtable_index;
204 bool _separate_io_proj;
205 CallDynamicJavaNode* _call_node;
206
207 protected:
208 void set_call_node(CallDynamicJavaNode* call) { _call_node = call; }
209
210 public:
211 VirtualCallGenerator(ciMethod* method, int vtable_index, bool separate_io_proj)
212 : CallGenerator(method), _vtable_index(vtable_index), _separate_io_proj(separate_io_proj), _call_node(NULL)
213 {
214 assert(vtable_index == Method::invalid_vtable_index ||
215 vtable_index >= 0, "either invalid or usable");
216 }
217 virtual bool is_virtual() const { return true; }
218 virtual JVMState* generate(JVMState* jvms);
219
220 virtual CallNode* call_node() const { return _call_node; }
221 int vtable_index() const { return _vtable_index; }
222
223 virtual CallGenerator* with_call_node(CallNode* call) {
224 VirtualCallGenerator* cg = new VirtualCallGenerator(method(), _vtable_index, _separate_io_proj);
225 cg->set_call_node(call->as_CallDynamicJava());
226 return cg;
227 }
228 };
229
230 JVMState* VirtualCallGenerator::generate(JVMState* jvms) {
231 GraphKit kit(jvms);
232 Node* receiver = kit.argument(0);
233 kit.C->print_inlining_update(this);
234
235 if (kit.C->log() != NULL) {
236 kit.C->log()->elem("virtual_call bci='%d'", jvms->bci());
237 }
238
239 // If the receiver is a constant null, do not torture the system
240 // by attempting to call through it. The compile will proceed
241 // correctly, but may bail out in final_graph_reshaping, because
242 // the call instruction will have a seemingly deficient out-count.
243 // (The bailout says something misleading about an "infinite loop".)
244 if (!receiver->is_InlineType() && kit.gvn().type(receiver)->higher_equal(TypePtr::NULL_PTR)) {
245 assert(Bytecodes::is_invoke(kit.java_bc()), "%d: %s", kit.java_bc(), Bytecodes::name(kit.java_bc()));
246 ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci());
247 int arg_size = declared_method->signature()->arg_size_for_bc(kit.java_bc());
248 kit.inc_sp(arg_size); // restore arguments
249 kit.uncommon_trap(Deoptimization::Reason_null_check,
250 Deoptimization::Action_none,
251 NULL, "null receiver");
252 return kit.transfer_exceptions_into_jvms();
253 }
254
255 // Ideally we would unconditionally do a null check here and let it
256 // be converted to an implicit check based on profile information.
257 // However currently the conversion to implicit null checks in
258 // Block::implicit_null_check() only looks for loads and stores, not calls.
259 ciMethod *caller = kit.method();
260 ciMethodData *caller_md = (caller == NULL) ? NULL : caller->method_data();
261 if (!UseInlineCaches || !ImplicitNullChecks || !os::zero_page_read_protected() ||
262 ((ImplicitNullCheckThreshold > 0) && caller_md &&
263 (caller_md->trap_count(Deoptimization::Reason_null_check)
264 >= (uint)ImplicitNullCheckThreshold))) {
272 }
273
274 assert(!method()->is_static(), "virtual call must not be to static");
275 assert(!method()->is_final(), "virtual call should not be to final");
276 assert(!method()->is_private(), "virtual call should not be to private");
277 assert(_vtable_index == Method::invalid_vtable_index || !UseInlineCaches,
278 "no vtable calls if +UseInlineCaches ");
279 address target = SharedRuntime::get_resolve_virtual_call_stub();
280 // Normal inline cache used for call
281 CallDynamicJavaNode* call = new CallDynamicJavaNode(tf(), target, method(), _vtable_index);
282 if (is_inlined_method_handle_intrinsic(jvms, method())) {
283 // To be able to issue a direct call (optimized virtual or virtual)
284 // and skip a call to MH.linkTo*/invokeBasic adapter, additional information
285 // about the method being invoked should be attached to the call site to
286 // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C).
287 call->set_override_symbolic_info(true);
288 }
289 _call_node = call; // Save the call node in case we need it later
290
291 kit.set_arguments_for_java_call(call);
292 if (kit.stopped()) {
293 return kit.transfer_exceptions_into_jvms();
294 }
295 kit.set_edges_for_java_call(call, false /*must_throw*/, _separate_io_proj);
296 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
297 kit.push_node(method()->return_type()->basic_type(), ret);
298
299 // Represent the effect of an implicit receiver null_check
300 // as part of this call. Since we share a map with the caller,
301 // his JVMS gets adjusted.
302 kit.cast_not_null(receiver);
303 return kit.transfer_exceptions_into_jvms();
304 }
305
306 CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) {
307 if (InlineTree::check_can_parse(m) != NULL) return NULL;
308 return new ParseGenerator(m, expected_uses);
309 }
310
311 // As a special case, the JVMS passed to this CallGenerator is
312 // for the method execution already in progress, not just the JVMS
313 // of the caller. Thus, this CallGenerator cannot be mixed with others!
314 CallGenerator* CallGenerator::for_osr(ciMethod* m, int osr_bci) {
368 return DirectCallGenerator::generate(jvms);
369 }
370
371 virtual void print_inlining_late(const char* msg) {
372 CallNode* call = call_node();
373 Compile* C = Compile::current();
374 C->print_inlining_assert_ready();
375 C->print_inlining(method(), call->jvms()->depth()-1, call->jvms()->bci(), msg);
376 C->print_inlining_move_to(this);
377 C->print_inlining_update_delayed(this);
378 }
379
380 virtual void set_unique_id(jlong id) {
381 _unique_id = id;
382 }
383
384 virtual jlong unique_id() const {
385 return _unique_id;
386 }
387
388 virtual CallGenerator* inline_cg() {
389 return _inline_cg;
390 }
391
392 virtual CallGenerator* with_call_node(CallNode* call) {
393 LateInlineCallGenerator* cg = new LateInlineCallGenerator(method(), _inline_cg, _is_pure_call);
394 cg->set_call_node(call->as_CallStaticJava());
395 return cg;
396 }
397 };
398
399 CallGenerator* CallGenerator::for_late_inline(ciMethod* method, CallGenerator* inline_cg) {
400 return new LateInlineCallGenerator(method, inline_cg);
401 }
402
403 class LateInlineMHCallGenerator : public LateInlineCallGenerator {
404 ciMethod* _caller;
405 bool _input_not_const;
406
407 virtual bool do_late_inline_check(Compile* C, JVMState* jvms);
408
409 public:
410 LateInlineMHCallGenerator(ciMethod* caller, ciMethod* callee, bool input_not_const) :
411 LateInlineCallGenerator(callee, NULL), _caller(caller), _input_not_const(input_not_const) {}
426 C->add_late_inline(this);
427 }
428 return new_jvms;
429 }
430
431 virtual CallGenerator* with_call_node(CallNode* call) {
432 LateInlineMHCallGenerator* cg = new LateInlineMHCallGenerator(_caller, method(), _input_not_const);
433 cg->set_call_node(call->as_CallStaticJava());
434 return cg;
435 }
436 };
437
438 bool LateInlineMHCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) {
439 // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call.
440 bool allow_inline = C->inlining_incrementally();
441 bool input_not_const = true;
442 CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), allow_inline, input_not_const);
443 assert(!input_not_const, "sanity"); // shouldn't have been scheduled for inlining in the first place
444
445 if (cg != NULL) {
446 // AlwaysIncrementalInline causes for_method_handle_inline() to
447 // return a LateInlineCallGenerator. Extract the
448 // InlineCallGenerato from it.
449 if (AlwaysIncrementalInline && cg->is_late_inline()) {
450 cg = cg->inline_cg();
451 }
452
453 assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline, "we're doing late inlining");
454 _inline_cg = cg;
455 C->dec_number_of_mh_late_inlines();
456 return true;
457 } else {
458 // Method handle call which has a constant appendix argument should be either inlined or replaced with a direct call
459 // unless there's a signature mismatch between caller and callee. If the failure occurs, there's not much to be improved later,
460 // so don't reinstall the generator to avoid pushing the generator between IGVN and incremental inlining indefinitely.
461 return false;
462 }
463 }
464
465 CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const) {
466 assert(IncrementalInlineMH, "required");
467 Compile::current()->inc_number_of_mh_late_inlines();
468 CallGenerator* cg = new LateInlineMHCallGenerator(caller, callee, input_not_const);
469 return cg;
470 }
471
472 // Allow inlining decisions to be delayed
643
644 #ifndef PRODUCT
645 if (PrintEliminateAllocations) {
646 tty->print("++++ Eliminated: %d ", call->_idx);
647 call->as_CallStaticJava()->method()->print_short_name(tty);
648 tty->cr();
649 }
650 #endif
651 }
652
653 void CallGenerator::do_late_inline_helper() {
654 assert(is_late_inline(), "only late inline allowed");
655
656 // Can't inline it
657 CallNode* call = call_node();
658 if (call == NULL || call->outcnt() == 0 ||
659 call->in(0) == NULL || call->in(0)->is_top()) {
660 return;
661 }
662
663 const TypeTuple* r = call->tf()->domain_cc();
664 for (uint i1 = TypeFunc::Parms; i1 < r->cnt(); i1++) {
665 if (call->in(i1)->is_top() && r->field_at(i1) != Type::HALF) {
666 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
667 return;
668 }
669 }
670
671 if (call->in(TypeFunc::Memory)->is_top()) {
672 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
673 return;
674 }
675 if (call->in(TypeFunc::Memory)->is_MergeMem()) {
676 MergeMemNode* merge_mem = call->in(TypeFunc::Memory)->as_MergeMem();
677 if (merge_mem->base_memory() == merge_mem->empty_memory()) {
678 return; // dead path
679 }
680 }
681
682 // check for unreachable loop
683 CallProjections* callprojs = call->extract_projections(true);
684 if ((callprojs->fallthrough_catchproj == call->in(0)) ||
685 (callprojs->catchall_catchproj == call->in(0)) ||
686 (callprojs->fallthrough_memproj == call->in(TypeFunc::Memory)) ||
687 (callprojs->catchall_memproj == call->in(TypeFunc::Memory)) ||
688 (callprojs->fallthrough_ioproj == call->in(TypeFunc::I_O)) ||
689 (callprojs->catchall_ioproj == call->in(TypeFunc::I_O)) ||
690 (callprojs->exobj != NULL && call->find_edge(callprojs->exobj) != -1)) {
691 return;
692 }
693
694 Compile* C = Compile::current();
695 // Remove inlined methods from Compiler's lists.
696 if (call->is_macro()) {
697 C->remove_macro_node(call);
698 }
699
700 bool result_not_used = false;
701
702 if (is_pure_call()) {
703 if (is_boxing_late_inline() && callprojs->resproj[0] != nullptr) {
704 // replace box node to scalar node only in case it is directly referenced by debug info
705 assert(call->as_CallStaticJava()->is_boxing_method(), "sanity");
706 if (!has_non_debug_usages(callprojs->resproj[0]) && is_box_cache_valid(call)) {
707 scalarize_debug_usages(call, callprojs->resproj[0]);
708 }
709 }
710
711 // The call is marked as pure (no important side effects), but result isn't used.
712 // It's safe to remove the call.
713 result_not_used = true;
714 for (uint i = 0; i < callprojs->nb_resproj; i++) {
715 if (callprojs->resproj[i] != NULL) {
716 if (callprojs->resproj[i]->outcnt() != 0) {
717 result_not_used = false;
718 }
719 if (call->find_edge(callprojs->resproj[i]) != -1) {
720 return;
721 }
722 }
723 }
724 }
725
726 if (result_not_used) {
727 GraphKit kit(call->jvms());
728 kit.replace_call(call, C->top(), true);
729 } else {
730 // Make a clone of the JVMState that appropriate to use for driving a parse
731 JVMState* old_jvms = call->jvms();
732 JVMState* jvms = old_jvms->clone_shallow(C);
733 uint size = call->req();
734 SafePointNode* map = new SafePointNode(size, jvms);
735 for (uint i1 = 0; i1 < size; i1++) {
736 map->init_req(i1, call->in(i1));
737 }
738
739 PhaseGVN& gvn = *C->initial_gvn();
740 // Make sure the state is a MergeMem for parsing.
741 if (!map->in(TypeFunc::Memory)->is_MergeMem()) {
742 Node* mem = MergeMemNode::make(map->in(TypeFunc::Memory));
743 gvn.set_type_bottom(mem);
744 map->set_req(TypeFunc::Memory, mem);
745 }
746
747 // blow away old call arguments
748 for (uint i1 = TypeFunc::Parms; i1 < r->cnt(); i1++) {
749 map->set_req(i1, C->top());
750 }
751 jvms->set_map(map);
752
753 // Make enough space in the expression stack to transfer
754 // the incoming arguments and return value.
755 map->ensure_stack(jvms, jvms->method()->max_stack());
756 const TypeTuple* domain_sig = call->_tf->domain_sig();
757 uint nargs = method()->arg_size();
758 assert(domain_sig->cnt() - TypeFunc::Parms == nargs, "inconsistent signature");
759
760 uint j = TypeFunc::Parms;
761 for (uint i1 = 0; i1 < nargs; i1++) {
762 const Type* t = domain_sig->field_at(TypeFunc::Parms + i1);
763 if (method()->has_scalarized_args() && t->is_inlinetypeptr() && !t->maybe_null() && t->inline_klass()->can_be_passed_as_fields()) {
764 // Inline type arguments are not passed by reference: we get an argument per
765 // field of the inline type. Build InlineTypeNodes from the inline type arguments.
766 GraphKit arg_kit(jvms, &gvn);
767 InlineTypeNode* vt = InlineTypeNode::make_from_multi(&arg_kit, call, t->inline_klass(), j, true);
768 map->set_control(arg_kit.control());
769 map->set_argument(jvms, i1, vt);
770 } else {
771 map->set_argument(jvms, i1, call->in(j++));
772 }
773 }
774
775 C->print_inlining_assert_ready();
776
777 C->print_inlining_move_to(this);
778
779 C->log_late_inline(this);
780
781 // JVMState is ready, so time to perform some checks and prepare for inlining attempt.
782 if (!do_late_inline_check(C, jvms)) {
783 map->disconnect_inputs(C);
784 C->print_inlining_update_delayed(this);
785 return;
786 }
787
788 // Check if we are late inlining a method handle call that returns an inline type as fields.
789 Node* buffer_oop = NULL;
790 ciType* mh_rt = inline_cg()->method()->return_type();
791 if (is_mh_late_inline() && mh_rt->is_inlinetype() && mh_rt->as_inline_klass()->can_be_returned_as_fields()) {
792 // Allocate a buffer for the inline type returned as fields because the caller expects an oop return.
793 // Do this before the method handle call in case the buffer allocation triggers deoptimization and
794 // we need to "re-execute" the call in the interpreter (to make sure the call is only executed once).
795 GraphKit arg_kit(jvms, &gvn);
796 {
797 PreserveReexecuteState preexecs(&arg_kit);
798 arg_kit.jvms()->set_should_reexecute(true);
799 arg_kit.inc_sp(nargs);
800 Node* klass_node = arg_kit.makecon(TypeKlassPtr::make(mh_rt->as_inline_klass()));
801 buffer_oop = arg_kit.new_instance(klass_node, NULL, NULL, /* deoptimize_on_exception */ true);
802 }
803 jvms = arg_kit.transfer_exceptions_into_jvms();
804 }
805
806 // Setup default node notes to be picked up by the inlining
807 Node_Notes* old_nn = C->node_notes_at(call->_idx);
808 if (old_nn != NULL) {
809 Node_Notes* entry_nn = old_nn->clone(C);
810 entry_nn->set_jvms(jvms);
811 C->set_default_node_notes(entry_nn);
812 }
813
814 // Now perform the inlining using the synthesized JVMState
815 JVMState* new_jvms = inline_cg()->generate(jvms);
816 if (new_jvms == NULL) return; // no change
817 if (C->failing()) return;
818
819 // Capture any exceptional control flow
820 GraphKit kit(new_jvms);
821
822 // Find the result object
823 Node* result = C->top();
824 int result_size = method()->return_type()->size();
825 if (result_size != 0 && !kit.stopped()) {
826 result = (result_size == 1) ? kit.pop() : kit.pop_pair();
827 }
828
829 if (inline_cg()->is_inline()) {
830 C->set_has_loops(C->has_loops() || inline_cg()->method()->has_loops());
831 C->env()->notice_inlined_method(inline_cg()->method());
832 }
833 C->set_inlining_progress(true);
834 C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup
835
836 // Handle inline type returns
837 InlineTypeNode* vt = result->isa_InlineType();
838 if (vt != NULL) {
839 if (call->tf()->returns_inline_type_as_fields()) {
840 vt->replace_call_results(&kit, call, C);
841 } else {
842 // Result might still be allocated (for example, if it has been stored to a non-flattened field)
843 if (!vt->is_allocated(&kit.gvn())) {
844 assert(buffer_oop != NULL, "should have allocated a buffer");
845 vt->store(&kit, buffer_oop, buffer_oop, vt->type()->inline_klass());
846 // Do not let stores that initialize this buffer be reordered with a subsequent
847 // store that would make this buffer accessible by other threads.
848 AllocateNode* alloc = AllocateNode::Ideal_allocation(buffer_oop, &kit.gvn());
849 assert(alloc != NULL, "must have an allocation node");
850 kit.insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress));
851 kit.gvn().hash_delete(vt);
852 vt->set_oop(buffer_oop);
853 vt = kit.gvn().transform(vt)->as_InlineType();
854 }
855 DEBUG_ONLY(buffer_oop = NULL);
856 // Convert to InlineTypePtrNode to keep track of field values
857 result = vt->as_ptr(&kit.gvn());
858 }
859 }
860 assert(buffer_oop == NULL, "unused buffer allocation");
861
862 kit.replace_call(call, result, true);
863 }
864 }
865
866 class LateInlineStringCallGenerator : public LateInlineCallGenerator {
867
868 public:
869 LateInlineStringCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
870 LateInlineCallGenerator(method, inline_cg) {}
871
872 virtual JVMState* generate(JVMState* jvms) {
873 Compile *C = Compile::current();
874
875 C->log_inline_id(this);
876
877 C->add_string_late_inline(this);
878
879 JVMState* new_jvms = DirectCallGenerator::generate(jvms);
880 return new_jvms;
881 }
1066 // Inline failed, so make a direct call.
1067 assert(_if_hit->is_inline(), "must have been a failed inline");
1068 CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method());
1069 new_jvms = cg->generate(kit.sync_jvms());
1070 }
1071 kit.add_exception_states_from(new_jvms);
1072 kit.set_jvms(new_jvms);
1073
1074 // Need to merge slow and fast?
1075 if (slow_map == NULL) {
1076 // The fast path is the only path remaining.
1077 return kit.transfer_exceptions_into_jvms();
1078 }
1079
1080 if (kit.stopped()) {
1081 // Inlined method threw an exception, so it's just the slow path after all.
1082 kit.set_jvms(slow_jvms);
1083 return kit.transfer_exceptions_into_jvms();
1084 }
1085
1086 // Allocate inline types if they are merged with objects (similar to Parse::merge_common())
1087 uint tos = kit.jvms()->stkoff() + kit.sp();
1088 uint limit = slow_map->req();
1089 for (uint i = TypeFunc::Parms; i < limit; i++) {
1090 Node* m = kit.map()->in(i);
1091 Node* n = slow_map->in(i);
1092 const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
1093 if (m->is_InlineType() && !t->isa_inlinetype()) {
1094 // Allocate inline type in fast path
1095 m = m->as_InlineType()->buffer(&kit);
1096 kit.map()->set_req(i, m);
1097 }
1098 if (n->is_InlineType() && !t->isa_inlinetype()) {
1099 // Allocate inline type in slow path
1100 PreserveJVMState pjvms(&kit);
1101 kit.set_map(slow_map);
1102 n = n->as_InlineType()->buffer(&kit);
1103 kit.map()->set_req(i, n);
1104 slow_map = kit.stop();
1105 }
1106 }
1107
1108 // There are 2 branches and the replaced nodes are only valid on
1109 // one: restore the replaced nodes to what they were before the
1110 // branch.
1111 kit.map()->set_replaced_nodes(replaced_nodes);
1112
1113 // Finish the diamond.
1114 kit.C->set_has_split_ifs(true); // Has chance for split-if optimization
1115 RegionNode* region = new RegionNode(3);
1116 region->init_req(1, kit.control());
1117 region->init_req(2, slow_map->control());
1118 kit.set_control(gvn.transform(region));
1119 Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO);
1120 iophi->set_req(2, slow_map->i_o());
1121 kit.set_i_o(gvn.transform(iophi));
1122 // Merge memory
1123 kit.merge_memory(slow_map->merged_memory(), region, 2);
1124 // Transform new memory Phis.
1125 for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) {
1126 Node* phi = mms.memory();
1127 if (phi->is_Phi() && phi->in(0) == region) {
1128 mms.set_memory(gvn.transform(phi));
1129 }
1130 }
1131 for (uint i = TypeFunc::Parms; i < limit; i++) {
1132 // Skip unused stack slots; fast forward to monoff();
1133 if (i == tos) {
1134 i = kit.jvms()->monoff();
1135 if( i >= limit ) break;
1136 }
1137 Node* m = kit.map()->in(i);
1138 Node* n = slow_map->in(i);
1139 if (m != n) {
1140 const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
1141 Node* phi = PhiNode::make(region, m, t);
1142 phi->set_req(2, n);
1143 kit.map()->set_req(i, gvn.transform(phi));
1144 }
1145 }
1146 return kit.transfer_exceptions_into_jvms();
1147 }
1148
1149
1150 CallGenerator* CallGenerator::for_method_handle_call(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline) {
1151 assert(callee->is_method_handle_intrinsic(), "for_method_handle_call mismatch");
1152 bool input_not_const;
1153 CallGenerator* cg = CallGenerator::for_method_handle_inline(jvms, caller, callee, allow_inline, input_not_const);
1154 Compile* C = Compile::current();
1155 if (cg != NULL) {
1156 if (AlwaysIncrementalInline) {
1157 return CallGenerator::for_late_inline(callee, cg);
1158 } else {
1159 return cg;
1160 }
1161 }
1162 int bci = jvms->bci();
1163 ciCallProfile profile = caller->call_profile_at_bci(bci);
1164 int call_site_count = caller->scale_count(profile.count());
1165
1166 if (IncrementalInlineMH && (AlwaysIncrementalInline ||
1167 (call_site_count > 0 && (input_not_const || !C->inlining_incrementally() || C->over_inlining_cutoff())))) {
1168 return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
1169 } else {
1170 // Out-of-line call.
1171 return CallGenerator::for_direct_call(callee);
1172 }
1173 }
1174
1175 static void cast_argument(int nargs, int arg_nb, ciType* t, GraphKit& kit, bool null_free) {
1176 PhaseGVN& gvn = kit.gvn();
1177 Node* arg = kit.argument(arg_nb);
1178 const Type* arg_type = arg->bottom_type();
1179 const Type* sig_type = TypeOopPtr::make_from_klass(t->as_klass());
1180 if (t->as_klass()->is_inlinetype() && null_free) {
1181 sig_type = sig_type->filter_speculative(TypePtr::NOTNULL);
1182 }
1183 if (arg_type->isa_oopptr() && !arg_type->higher_equal(sig_type)) {
1184 const Type* narrowed_arg_type = arg_type->filter_speculative(sig_type); // keep speculative part
1185 arg = gvn.transform(new CheckCastPPNode(kit.control(), arg, narrowed_arg_type));
1186 kit.set_argument(arg_nb, arg);
1187 }
1188 if (sig_type->is_inlinetypeptr() && !arg->is_InlineType()) {
1189 arg = InlineTypeNode::make_from_oop(&kit, arg, t->as_inline_klass(), !kit.gvn().type(arg)->maybe_null());
1190 kit.set_argument(arg_nb, arg);
1191 }
1192 }
1193
1194 class NativeCallGenerator : public CallGenerator {
1195 private:
1196 address _call_addr;
1197 ciNativeEntryPoint* _nep;
1198 public:
1199 NativeCallGenerator(ciMethod* m, address call_addr, ciNativeEntryPoint* nep)
1200 : CallGenerator(m), _call_addr(call_addr), _nep(nep) {}
1201
1202 virtual JVMState* generate(JVMState* jvms);
1203 };
1204
1205 JVMState* NativeCallGenerator::generate(JVMState* jvms) {
1206 GraphKit kit(jvms);
1207
1208 Node* call = kit.make_native_call(_call_addr, tf(), method()->arg_size(), _nep); // -fallback, - nep
1209 if (call == NULL) return NULL;
1210
1211 kit.C->print_inlining_update(this);
1212 if (kit.C->log() != NULL) {
1213 kit.C->log()->elem("l2n_intrinsification_success bci='%d' entry_point='" INTPTR_FORMAT "'", jvms->bci(), p2i(_call_addr));
1228 switch (iid) {
1229 case vmIntrinsics::_invokeBasic:
1230 {
1231 // Get MethodHandle receiver:
1232 Node* receiver = kit.argument(0);
1233 if (receiver->Opcode() == Op_ConP) {
1234 input_not_const = false;
1235 const TypeOopPtr* oop_ptr = receiver->bottom_type()->is_oopptr();
1236 ciMethod* target = oop_ptr->const_oop()->as_method_handle()->get_vmtarget();
1237 const int vtable_index = Method::invalid_vtable_index;
1238
1239 if (!ciMethod::is_consistent_info(callee, target)) {
1240 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1241 "signatures mismatch");
1242 return NULL;
1243 }
1244
1245 CallGenerator* cg = C->call_generator(target, vtable_index,
1246 false /* call_does_dispatch */,
1247 jvms,
1248 true /* allow_inline */,
1249 PROB_ALWAYS,
1250 NULL,
1251 true);
1252 return cg;
1253 } else {
1254 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1255 "receiver not constant");
1256 }
1257 }
1258 break;
1259
1260 case vmIntrinsics::_linkToVirtual:
1261 case vmIntrinsics::_linkToStatic:
1262 case vmIntrinsics::_linkToSpecial:
1263 case vmIntrinsics::_linkToInterface:
1264 {
1265 int nargs = callee->arg_size();
1266 // Get MemberName argument:
1267 Node* member_name = kit.argument(nargs - 1);
1268 if (member_name->Opcode() == Op_ConP) {
1269 input_not_const = false;
1270 const TypeOopPtr* oop_ptr = member_name->bottom_type()->is_oopptr();
1271 ciMethod* target = oop_ptr->const_oop()->as_member_name()->get_vmtarget();
1272
1273 if (!ciMethod::is_consistent_info(callee, target)) {
1274 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1275 "signatures mismatch");
1276 return NULL;
1277 }
1278
1279 // In lambda forms we erase signature types to avoid resolving issues
1280 // involving class loaders. When we optimize a method handle invoke
1281 // to a direct call we must cast the receiver and arguments to its
1282 // actual types.
1283 ciSignature* signature = target->signature();
1284 const int receiver_skip = target->is_static() ? 0 : 1;
1285 // Cast receiver to its type.
1286 if (!target->is_static()) {
1287 cast_argument(nargs, 0, signature->accessing_klass(), kit, false);
1288 }
1289 // Cast reference arguments to its type.
1290 for (int i = 0, j = 0; i < signature->count(); i++) {
1291 ciType* t = signature->type_at(i);
1292 if (t->is_klass()) {
1293 bool null_free = signature->is_null_free_at(i);
1294 cast_argument(nargs, receiver_skip + j, t, kit, null_free);
1295 }
1296 j += t->size(); // long and double take two slots
1297 }
1298
1299 // Try to get the most accurate receiver type
1300 const bool is_virtual = (iid == vmIntrinsics::_linkToVirtual);
1301 const bool is_virtual_or_interface = (is_virtual || iid == vmIntrinsics::_linkToInterface);
1302 int vtable_index = Method::invalid_vtable_index;
1303 bool call_does_dispatch = false;
1304
1305 ciKlass* speculative_receiver_type = NULL;
1306 if (is_virtual_or_interface) {
1307 ciInstanceKlass* klass = target->holder();
1308 Node* receiver_node = kit.argument(0);
1309 const TypeOopPtr* receiver_type = gvn.type(receiver_node)->isa_oopptr();
1310 // call_does_dispatch and vtable_index are out-parameters. They might be changed.
1311 // optimize_virtual_call() takes 2 different holder
1312 // arguments for a corner case that doesn't apply here (see
1313 // Parse::do_call())
1314 target = C->optimize_virtual_call(caller, klass, klass,
1315 target, receiver_type, is_virtual,
1316 call_does_dispatch, vtable_index, // out-parameters
1317 false /* check_access */);
1318 // We lack profiling at this call but type speculation may
1319 // provide us with a type
1320 speculative_receiver_type = (receiver_type != NULL) ? receiver_type->speculative_type() : NULL;
1321 }
1322 CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms,
1323 allow_inline,
1324 PROB_ALWAYS,
1325 speculative_receiver_type,
1326 true);
1327 return cg;
1328 } else {
1329 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1330 "member_name not constant");
1331 }
1332 }
1333 break;
1334
1335 case vmIntrinsics::_linkToNative:
1336 {
1337 Node* addr_n = kit.argument(1); // target address
1338 Node* nep_n = kit.argument(callee->arg_size() - 1); // NativeEntryPoint
1339 // This check needs to be kept in sync with the one in CallStaticJavaNode::Ideal
1340 if (addr_n->Opcode() == Op_ConL && nep_n->Opcode() == Op_ConP) {
1341 input_not_const = false;
1342 const TypeLong* addr_t = addr_n->bottom_type()->is_long();
1343 const TypeOopPtr* nep_t = nep_n->bottom_type()->is_oopptr();
1344 address addr = (address) addr_t->get_con();
1345 ciNativeEntryPoint* nep = nep_t->const_oop()->as_native_entry_point();
1346 return new NativeCallGenerator(callee, addr, nep);
1396 // do_intrinsic(0)
1397 // else
1398 // if (predicate(1))
1399 // do_intrinsic(1)
1400 // ...
1401 // else
1402 // do_java_comp
1403
1404 GraphKit kit(jvms);
1405 PhaseGVN& gvn = kit.gvn();
1406
1407 CompileLog* log = kit.C->log();
1408 if (log != NULL) {
1409 log->elem("predicated_intrinsic bci='%d' method='%d'",
1410 jvms->bci(), log->identify(method()));
1411 }
1412
1413 if (!method()->is_static()) {
1414 // We need an explicit receiver null_check before checking its type in predicate.
1415 // We share a map with the caller, so his JVMS gets adjusted.
1416 kit.null_check_receiver_before_call(method());
1417 if (kit.stopped()) {
1418 return kit.transfer_exceptions_into_jvms();
1419 }
1420 }
1421
1422 int n_predicates = _intrinsic->predicates_count();
1423 assert(n_predicates > 0, "sanity");
1424
1425 JVMState** result_jvms = NEW_RESOURCE_ARRAY(JVMState*, (n_predicates+1));
1426
1427 // Region for normal compilation code if intrinsic failed.
1428 Node* slow_region = new RegionNode(1);
1429
1430 int results = 0;
1431 for (int predicate = 0; (predicate < n_predicates) && !kit.stopped(); predicate++) {
1432 #ifdef ASSERT
1433 JVMState* old_jvms = kit.jvms();
1434 SafePointNode* old_map = kit.map();
1435 Node* old_io = old_map->i_o();
1436 Node* old_mem = old_map->memory();
|