18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/bcEscapeAnalyzer.hpp"
27 #include "ci/ciCallSite.hpp"
28 #include "ci/ciObjArray.hpp"
29 #include "ci/ciMemberName.hpp"
30 #include "ci/ciMethodHandle.hpp"
31 #include "classfile/javaClasses.hpp"
32 #include "compiler/compileLog.hpp"
33 #include "opto/addnode.hpp"
34 #include "opto/callGenerator.hpp"
35 #include "opto/callnode.hpp"
36 #include "opto/castnode.hpp"
37 #include "opto/cfgnode.hpp"
38 #include "opto/parse.hpp"
39 #include "opto/rootnode.hpp"
40 #include "opto/runtime.hpp"
41 #include "opto/subnode.hpp"
42 #include "runtime/os.inline.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "utilities/debug.hpp"
45
46 // Utility function.
47 const TypeFunc* CallGenerator::tf() const {
48 return TypeFunc::make(method());
49 }
50
51 bool CallGenerator::is_inlined_method_handle_intrinsic(JVMState* jvms, ciMethod* m) {
52 return is_inlined_method_handle_intrinsic(jvms->method(), jvms->bci(), m);
53 }
54
55 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* caller, int bci, ciMethod* m) {
56 ciMethod* symbolic_info = caller->get_method_at_bci(bci);
57 return is_inlined_method_handle_intrinsic(symbolic_info, m);
101 GraphKit& exits = parser.exits();
102
103 if (C->failing()) {
104 while (exits.pop_exception_state() != NULL) ;
105 return NULL;
106 }
107
108 assert(exits.jvms()->same_calls_as(jvms), "sanity");
109
110 // Simply return the exit state of the parser,
111 // augmented by any exceptional states.
112 return exits.transfer_exceptions_into_jvms();
113 }
114
115 //---------------------------DirectCallGenerator------------------------------
116 // Internal class which handles all out-of-line calls w/o receiver type checks.
117 class DirectCallGenerator : public CallGenerator {
118 private:
119 CallStaticJavaNode* _call_node;
120 // Force separate memory and I/O projections for the exceptional
121 // paths to facilitate late inlinig.
122 bool _separate_io_proj;
123
124 protected:
125 void set_call_node(CallStaticJavaNode* call) { _call_node = call; }
126
127 public:
128 DirectCallGenerator(ciMethod* method, bool separate_io_proj)
129 : CallGenerator(method),
130 _separate_io_proj(separate_io_proj)
131 {
132 }
133 virtual JVMState* generate(JVMState* jvms);
134
135 virtual CallNode* call_node() const { return _call_node; }
136 virtual CallGenerator* with_call_node(CallNode* call) {
137 DirectCallGenerator* dcg = new DirectCallGenerator(method(), _separate_io_proj);
138 dcg->set_call_node(call->as_CallStaticJava());
139 return dcg;
140 }
141 };
142
143 JVMState* DirectCallGenerator::generate(JVMState* jvms) {
144 GraphKit kit(jvms);
145 kit.C->print_inlining_update(this);
146 bool is_static = method()->is_static();
147 address target = is_static ? SharedRuntime::get_resolve_static_call_stub()
148 : SharedRuntime::get_resolve_opt_virtual_call_stub();
149
150 if (kit.C->log() != NULL) {
151 kit.C->log()->elem("direct_call bci='%d'", jvms->bci());
152 }
153
154 CallStaticJavaNode* call = new CallStaticJavaNode(kit.C, tf(), target, method());
155 if (is_inlined_method_handle_intrinsic(jvms, method())) {
156 // To be able to issue a direct call and skip a call to MH.linkTo*/invokeBasic adapter,
157 // additional information about the method being invoked should be attached
158 // to the call site to make resolution logic work
159 // (see SharedRuntime::resolve_static_call_C).
160 call->set_override_symbolic_info(true);
161 }
162 _call_node = call; // Save the call node in case we need it later
163 if (!is_static) {
164 // Make an explicit receiver null_check as part of this call.
165 // Since we share a map with the caller, his JVMS gets adjusted.
166 kit.null_check_receiver_before_call(method());
167 if (kit.stopped()) {
168 // And dump it back to the caller, decorated with any exceptions:
169 return kit.transfer_exceptions_into_jvms();
170 }
171 // Mark the call node as virtual, sort of:
172 call->set_optimized_virtual(true);
173 if (method()->is_method_handle_intrinsic() ||
174 method()->is_compiled_lambda_form()) {
175 call->set_method_handle_invoke(true);
176 }
177 }
178 kit.set_arguments_for_java_call(call);
179 kit.set_edges_for_java_call(call, false, _separate_io_proj);
180 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
181 kit.push_node(method()->return_type()->basic_type(), ret);
182 return kit.transfer_exceptions_into_jvms();
183 }
184
185 //--------------------------VirtualCallGenerator------------------------------
186 // Internal class which handles all out-of-line calls checking receiver type.
187 class VirtualCallGenerator : public CallGenerator {
188 private:
189 int _vtable_index;
190 bool _separate_io_proj;
191 CallDynamicJavaNode* _call_node;
192
193 protected:
194 void set_call_node(CallDynamicJavaNode* call) { _call_node = call; }
195
196 public:
197 VirtualCallGenerator(ciMethod* method, int vtable_index, bool separate_io_proj)
198 : CallGenerator(method), _vtable_index(vtable_index), _separate_io_proj(separate_io_proj), _call_node(NULL)
199 {
200 assert(vtable_index == Method::invalid_vtable_index ||
201 vtable_index >= 0, "either invalid or usable");
202 }
203 virtual bool is_virtual() const { return true; }
204 virtual JVMState* generate(JVMState* jvms);
205
206 virtual CallNode* call_node() const { return _call_node; }
207 int vtable_index() const { return _vtable_index; }
208
209 virtual CallGenerator* with_call_node(CallNode* call) {
210 VirtualCallGenerator* cg = new VirtualCallGenerator(method(), _vtable_index, _separate_io_proj);
211 cg->set_call_node(call->as_CallDynamicJava());
212 return cg;
213 }
214 };
215
216 JVMState* VirtualCallGenerator::generate(JVMState* jvms) {
217 GraphKit kit(jvms);
218 Node* receiver = kit.argument(0);
219
220 kit.C->print_inlining_update(this);
221
222 if (kit.C->log() != NULL) {
223 kit.C->log()->elem("virtual_call bci='%d'", jvms->bci());
224 }
225
226 // If the receiver is a constant null, do not torture the system
227 // by attempting to call through it. The compile will proceed
228 // correctly, but may bail out in final_graph_reshaping, because
229 // the call instruction will have a seemingly deficient out-count.
230 // (The bailout says something misleading about an "infinite loop".)
231 if (kit.gvn().type(receiver)->higher_equal(TypePtr::NULL_PTR)) {
232 assert(Bytecodes::is_invoke(kit.java_bc()), "%d: %s", kit.java_bc(), Bytecodes::name(kit.java_bc()));
233 ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci());
234 int arg_size = declared_method->signature()->arg_size_for_bc(kit.java_bc());
235 kit.inc_sp(arg_size); // restore arguments
236 kit.uncommon_trap(Deoptimization::Reason_null_check,
237 Deoptimization::Action_none,
238 NULL, "null receiver");
239 return kit.transfer_exceptions_into_jvms();
240 }
241
242 // Ideally we would unconditionally do a null check here and let it
243 // be converted to an implicit check based on profile information.
244 // However currently the conversion to implicit null checks in
245 // Block::implicit_null_check() only looks for loads and stores, not calls.
246 ciMethod *caller = kit.method();
247 ciMethodData *caller_md = (caller == NULL) ? NULL : caller->method_data();
248 if (!UseInlineCaches || !ImplicitNullChecks || !os::zero_page_read_protected() ||
249 ((ImplicitNullCheckThreshold > 0) && caller_md &&
250 (caller_md->trap_count(Deoptimization::Reason_null_check)
251 >= (uint)ImplicitNullCheckThreshold))) {
259 }
260
261 assert(!method()->is_static(), "virtual call must not be to static");
262 assert(!method()->is_final(), "virtual call should not be to final");
263 assert(!method()->is_private(), "virtual call should not be to private");
264 assert(_vtable_index == Method::invalid_vtable_index || !UseInlineCaches,
265 "no vtable calls if +UseInlineCaches ");
266 address target = SharedRuntime::get_resolve_virtual_call_stub();
267 // Normal inline cache used for call
268 CallDynamicJavaNode* call = new CallDynamicJavaNode(tf(), target, method(), _vtable_index);
269 if (is_inlined_method_handle_intrinsic(jvms, method())) {
270 // To be able to issue a direct call (optimized virtual or virtual)
271 // and skip a call to MH.linkTo*/invokeBasic adapter, additional information
272 // about the method being invoked should be attached to the call site to
273 // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C).
274 call->set_override_symbolic_info(true);
275 }
276 _call_node = call; // Save the call node in case we need it later
277
278 kit.set_arguments_for_java_call(call);
279 kit.set_edges_for_java_call(call, false /*must_throw*/, _separate_io_proj);
280 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
281 kit.push_node(method()->return_type()->basic_type(), ret);
282
283 // Represent the effect of an implicit receiver null_check
284 // as part of this call. Since we share a map with the caller,
285 // his JVMS gets adjusted.
286 kit.cast_not_null(receiver);
287 return kit.transfer_exceptions_into_jvms();
288 }
289
290 CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) {
291 if (InlineTree::check_can_parse(m) != NULL) return NULL;
292 return new ParseGenerator(m, expected_uses);
293 }
294
295 // As a special case, the JVMS passed to this CallGenerator is
296 // for the method execution already in progress, not just the JVMS
297 // of the caller. Thus, this CallGenerator cannot be mixed with others!
298 CallGenerator* CallGenerator::for_osr(ciMethod* m, int osr_bci) {
352 return DirectCallGenerator::generate(jvms);
353 }
354
355 virtual void print_inlining_late(const char* msg) {
356 CallNode* call = call_node();
357 Compile* C = Compile::current();
358 C->print_inlining_assert_ready();
359 C->print_inlining(method(), call->jvms()->depth()-1, call->jvms()->bci(), msg);
360 C->print_inlining_move_to(this);
361 C->print_inlining_update_delayed(this);
362 }
363
364 virtual void set_unique_id(jlong id) {
365 _unique_id = id;
366 }
367
368 virtual jlong unique_id() const {
369 return _unique_id;
370 }
371
372 virtual CallGenerator* with_call_node(CallNode* call) {
373 LateInlineCallGenerator* cg = new LateInlineCallGenerator(method(), _inline_cg, _is_pure_call);
374 cg->set_call_node(call->as_CallStaticJava());
375 return cg;
376 }
377 };
378
379 CallGenerator* CallGenerator::for_late_inline(ciMethod* method, CallGenerator* inline_cg) {
380 return new LateInlineCallGenerator(method, inline_cg);
381 }
382
383 class LateInlineMHCallGenerator : public LateInlineCallGenerator {
384 ciMethod* _caller;
385 bool _input_not_const;
386
387 virtual bool do_late_inline_check(Compile* C, JVMState* jvms);
388
389 public:
390 LateInlineMHCallGenerator(ciMethod* caller, ciMethod* callee, bool input_not_const) :
391 LateInlineCallGenerator(callee, NULL), _caller(caller), _input_not_const(input_not_const) {}
413 cg->set_call_node(call->as_CallStaticJava());
414 return cg;
415 }
416 };
417
418 bool LateInlineMHCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) {
419 // When inlining a virtual call, the null check at the call and the call itself can throw. These 2 paths have different
420 // expression stacks which causes late inlining to break. The MH invoker is not expected to be called from a method with
421 // exception handlers. When there is no exception handler, GraphKit::builtin_throw() pops the stack which solves the issue
422 // of late inlining with exceptions.
423 assert(!jvms->method()->has_exception_handlers() ||
424 (method()->intrinsic_id() != vmIntrinsics::_linkToVirtual &&
425 method()->intrinsic_id() != vmIntrinsics::_linkToInterface), "no exception handler expected");
426 // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call.
427 bool allow_inline = C->inlining_incrementally();
428 bool input_not_const = true;
429 CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), allow_inline, input_not_const);
430 assert(!input_not_const, "sanity"); // shouldn't have been scheduled for inlining in the first place
431
432 if (cg != NULL) {
433 assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline, "we're doing late inlining");
434 _inline_cg = cg;
435 C->dec_number_of_mh_late_inlines();
436 return true;
437 } else {
438 // Method handle call which has a constant appendix argument should be either inlined or replaced with a direct call
439 // unless there's a signature mismatch between caller and callee. If the failure occurs, there's not much to be improved later,
440 // so don't reinstall the generator to avoid pushing the generator between IGVN and incremental inlining indefinitely.
441 return false;
442 }
443 }
444
445 CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const) {
446 assert(IncrementalInlineMH, "required");
447 Compile::current()->inc_number_of_mh_late_inlines();
448 CallGenerator* cg = new LateInlineMHCallGenerator(caller, callee, input_not_const);
449 return cg;
450 }
451
452 // Allow inlining decisions to be delayed
575
576 void LateInlineMHCallGenerator::do_late_inline() {
577 CallGenerator::do_late_inline_helper();
578 }
579
580 void LateInlineVirtualCallGenerator::do_late_inline() {
581 assert(_callee != NULL, "required"); // set up in CallDynamicJavaNode::Ideal
582 CallGenerator::do_late_inline_helper();
583 }
584
585 void CallGenerator::do_late_inline_helper() {
586 assert(is_late_inline(), "only late inline allowed");
587
588 // Can't inline it
589 CallNode* call = call_node();
590 if (call == NULL || call->outcnt() == 0 ||
591 call->in(0) == NULL || call->in(0)->is_top()) {
592 return;
593 }
594
595 const TypeTuple *r = call->tf()->domain();
596 for (int i1 = 0; i1 < method()->arg_size(); i1++) {
597 if (call->in(TypeFunc::Parms + i1)->is_top() && r->field_at(TypeFunc::Parms + i1) != Type::HALF) {
598 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
599 return;
600 }
601 }
602
603 if (call->in(TypeFunc::Memory)->is_top()) {
604 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
605 return;
606 }
607 if (call->in(TypeFunc::Memory)->is_MergeMem()) {
608 MergeMemNode* merge_mem = call->in(TypeFunc::Memory)->as_MergeMem();
609 if (merge_mem->base_memory() == merge_mem->empty_memory()) {
610 return; // dead path
611 }
612 }
613
614 // check for unreachable loop
615 CallProjections callprojs;
616 call->extract_projections(&callprojs, true);
617 if ((callprojs.fallthrough_catchproj == call->in(0)) ||
618 (callprojs.catchall_catchproj == call->in(0)) ||
619 (callprojs.fallthrough_memproj == call->in(TypeFunc::Memory)) ||
620 (callprojs.catchall_memproj == call->in(TypeFunc::Memory)) ||
621 (callprojs.fallthrough_ioproj == call->in(TypeFunc::I_O)) ||
622 (callprojs.catchall_ioproj == call->in(TypeFunc::I_O)) ||
623 (callprojs.resproj != NULL && call->find_edge(callprojs.resproj) != -1) ||
624 (callprojs.exobj != NULL && call->find_edge(callprojs.exobj) != -1)) {
625 return;
626 }
627
628 Compile* C = Compile::current();
629 // Remove inlined methods from Compiler's lists.
630 if (call->is_macro()) {
631 C->remove_macro_node(call);
632 }
633
634 // The call is marked as pure (no important side effects), but result isn't used.
635 // It's safe to remove the call.
636 bool result_not_used = (callprojs.resproj == NULL || callprojs.resproj->outcnt() == 0);
637
638 if (is_pure_call() && result_not_used) {
639 GraphKit kit(call->jvms());
640 kit.replace_call(call, C->top(), true);
641 } else {
642 // Make a clone of the JVMState that appropriate to use for driving a parse
643 JVMState* old_jvms = call->jvms();
644 JVMState* jvms = old_jvms->clone_shallow(C);
645 uint size = call->req();
646 SafePointNode* map = new SafePointNode(size, jvms);
647 for (uint i1 = 0; i1 < size; i1++) {
648 map->init_req(i1, call->in(i1));
649 }
650
651 // Make sure the state is a MergeMem for parsing.
652 if (!map->in(TypeFunc::Memory)->is_MergeMem()) {
653 Node* mem = MergeMemNode::make(map->in(TypeFunc::Memory));
654 C->initial_gvn()->set_type_bottom(mem);
655 map->set_req(TypeFunc::Memory, mem);
656 }
657
658 uint nargs = method()->arg_size();
659 // blow away old call arguments
660 Node* top = C->top();
661 for (uint i1 = 0; i1 < nargs; i1++) {
662 map->set_req(TypeFunc::Parms + i1, top);
663 }
664 jvms->set_map(map);
665
666 // Make enough space in the expression stack to transfer
667 // the incoming arguments and return value.
668 map->ensure_stack(jvms, jvms->method()->max_stack());
669 for (uint i1 = 0; i1 < nargs; i1++) {
670 map->set_argument(jvms, i1, call->in(TypeFunc::Parms + i1));
671 }
672
673 C->print_inlining_assert_ready();
674
675 C->print_inlining_move_to(this);
676
677 C->log_late_inline(this);
678
679 // JVMState is ready, so time to perform some checks and prepare for inlining attempt.
680 if (!do_late_inline_check(C, jvms)) {
681 map->disconnect_inputs(C);
682 C->print_inlining_update_delayed(this);
683 return;
684 }
685
686 // Setup default node notes to be picked up by the inlining
687 Node_Notes* old_nn = C->node_notes_at(call->_idx);
688 if (old_nn != NULL) {
689 Node_Notes* entry_nn = old_nn->clone(C);
690 entry_nn->set_jvms(jvms);
691 C->set_default_node_notes(entry_nn);
692 }
693
694 // Now perform the inlining using the synthesized JVMState
695 JVMState* new_jvms = inline_cg()->generate(jvms);
696 if (new_jvms == NULL) return; // no change
697 if (C->failing()) return;
698
699 // Capture any exceptional control flow
700 GraphKit kit(new_jvms);
701
702 // Find the result object
703 Node* result = C->top();
704 int result_size = method()->return_type()->size();
705 if (result_size != 0 && !kit.stopped()) {
706 result = (result_size == 1) ? kit.pop() : kit.pop_pair();
707 }
708
709 if (inline_cg()->is_inline()) {
710 C->set_has_loops(C->has_loops() || inline_cg()->method()->has_loops());
711 C->env()->notice_inlined_method(inline_cg()->method());
712 }
713 C->set_inlining_progress(true);
714 C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup
715 kit.replace_call(call, result, true);
716 }
717 }
718
719 class LateInlineStringCallGenerator : public LateInlineCallGenerator {
720
721 public:
722 LateInlineStringCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
723 LateInlineCallGenerator(method, inline_cg) {}
724
725 virtual JVMState* generate(JVMState* jvms) {
726 Compile *C = Compile::current();
727
728 C->log_inline_id(this);
729
730 C->add_string_late_inline(this);
731
732 JVMState* new_jvms = DirectCallGenerator::generate(jvms);
733 return new_jvms;
734 }
917 // Inline failed, so make a direct call.
918 assert(_if_hit->is_inline(), "must have been a failed inline");
919 CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method());
920 new_jvms = cg->generate(kit.sync_jvms());
921 }
922 kit.add_exception_states_from(new_jvms);
923 kit.set_jvms(new_jvms);
924
925 // Need to merge slow and fast?
926 if (slow_map == NULL) {
927 // The fast path is the only path remaining.
928 return kit.transfer_exceptions_into_jvms();
929 }
930
931 if (kit.stopped()) {
932 // Inlined method threw an exception, so it's just the slow path after all.
933 kit.set_jvms(slow_jvms);
934 return kit.transfer_exceptions_into_jvms();
935 }
936
937 // There are 2 branches and the replaced nodes are only valid on
938 // one: restore the replaced nodes to what they were before the
939 // branch.
940 kit.map()->set_replaced_nodes(replaced_nodes);
941
942 // Finish the diamond.
943 kit.C->set_has_split_ifs(true); // Has chance for split-if optimization
944 RegionNode* region = new RegionNode(3);
945 region->init_req(1, kit.control());
946 region->init_req(2, slow_map->control());
947 kit.set_control(gvn.transform(region));
948 Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO);
949 iophi->set_req(2, slow_map->i_o());
950 kit.set_i_o(gvn.transform(iophi));
951 // Merge memory
952 kit.merge_memory(slow_map->merged_memory(), region, 2);
953 // Transform new memory Phis.
954 for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) {
955 Node* phi = mms.memory();
956 if (phi->is_Phi() && phi->in(0) == region) {
957 mms.set_memory(gvn.transform(phi));
958 }
959 }
960 uint tos = kit.jvms()->stkoff() + kit.sp();
961 uint limit = slow_map->req();
962 for (uint i = TypeFunc::Parms; i < limit; i++) {
963 // Skip unused stack slots; fast forward to monoff();
964 if (i == tos) {
965 i = kit.jvms()->monoff();
966 if( i >= limit ) break;
967 }
968 Node* m = kit.map()->in(i);
969 Node* n = slow_map->in(i);
970 if (m != n) {
971 const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
972 Node* phi = PhiNode::make(region, m, t);
973 phi->set_req(2, n);
974 kit.map()->set_req(i, gvn.transform(phi));
975 }
976 }
977 return kit.transfer_exceptions_into_jvms();
978 }
979
980
981 CallGenerator* CallGenerator::for_method_handle_call(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline) {
982 assert(callee->is_method_handle_intrinsic(), "for_method_handle_call mismatch");
983 bool input_not_const;
984 CallGenerator* cg = CallGenerator::for_method_handle_inline(jvms, caller, callee, allow_inline, input_not_const);
985 Compile* C = Compile::current();
986 if (cg != NULL) {
987 if (AlwaysIncrementalInline) {
988 return CallGenerator::for_late_inline(callee, cg);
989 } else {
990 return cg;
991 }
992 }
993 int bci = jvms->bci();
994 ciCallProfile profile = caller->call_profile_at_bci(bci);
995 int call_site_count = caller->scale_count(profile.count());
996
997 if (IncrementalInlineMH && call_site_count > 0 &&
998 (input_not_const || !C->inlining_incrementally() || C->over_inlining_cutoff())) {
999 return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
1000 } else {
1001 // Out-of-line call.
1002 return CallGenerator::for_direct_call(callee);
1003 }
1004 }
1005
1006 CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline, bool& input_not_const) {
1007 GraphKit kit(jvms);
1008 PhaseGVN& gvn = kit.gvn();
1009 Compile* C = kit.C;
1010 vmIntrinsics::ID iid = callee->intrinsic_id();
1011 input_not_const = true;
1012 if (StressMethodHandleLinkerInlining) {
1013 allow_inline = false;
1014 }
1015 switch (iid) {
1016 case vmIntrinsics::_invokeBasic:
1017 {
1018 // Get MethodHandle receiver:
1019 Node* receiver = kit.argument(0);
1020 if (receiver->Opcode() == Op_ConP) {
1021 input_not_const = false;
1022 const TypeOopPtr* recv_toop = receiver->bottom_type()->isa_oopptr();
1023 if (recv_toop != NULL) {
1024 ciMethod* target = recv_toop->const_oop()->as_method_handle()->get_vmtarget();
1025 const int vtable_index = Method::invalid_vtable_index;
1037 PROB_ALWAYS);
1038 return cg;
1039 } else {
1040 assert(receiver->bottom_type() == TypePtr::NULL_PTR, "not a null: %s",
1041 Type::str(receiver->bottom_type()));
1042 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1043 "receiver is always null");
1044 }
1045 } else {
1046 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1047 "receiver not constant");
1048 }
1049 }
1050 break;
1051
1052 case vmIntrinsics::_linkToVirtual:
1053 case vmIntrinsics::_linkToStatic:
1054 case vmIntrinsics::_linkToSpecial:
1055 case vmIntrinsics::_linkToInterface:
1056 {
1057 // Get MemberName argument:
1058 Node* member_name = kit.argument(callee->arg_size() - 1);
1059 if (member_name->Opcode() == Op_ConP) {
1060 input_not_const = false;
1061 const TypeOopPtr* oop_ptr = member_name->bottom_type()->is_oopptr();
1062 ciMethod* target = oop_ptr->const_oop()->as_member_name()->get_vmtarget();
1063
1064 if (!ciMethod::is_consistent_info(callee, target)) {
1065 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1066 "signatures mismatch");
1067 return NULL;
1068 }
1069
1070 // In lambda forms we erase signature types to avoid resolving issues
1071 // involving class loaders. When we optimize a method handle invoke
1072 // to a direct call we must cast the receiver and arguments to its
1073 // actual types.
1074 ciSignature* signature = target->signature();
1075 const int receiver_skip = target->is_static() ? 0 : 1;
1076 // Cast receiver to its type.
1077 if (!target->is_static()) {
1078 Node* arg = kit.argument(0);
1079 const TypeOopPtr* arg_type = arg->bottom_type()->isa_oopptr();
1080 const Type* sig_type = TypeOopPtr::make_from_klass(signature->accessing_klass());
1081 if (arg_type != NULL && !arg_type->higher_equal(sig_type)) {
1082 const Type* recv_type = arg_type->filter_speculative(sig_type); // keep speculative part
1083 Node* cast_obj = gvn.transform(new CheckCastPPNode(kit.control(), arg, recv_type));
1084 kit.set_argument(0, cast_obj);
1085 }
1086 }
1087 // Cast reference arguments to its type.
1088 for (int i = 0, j = 0; i < signature->count(); i++) {
1089 ciType* t = signature->type_at(i);
1090 if (t->is_klass()) {
1091 Node* arg = kit.argument(receiver_skip + j);
1092 const TypeOopPtr* arg_type = arg->bottom_type()->isa_oopptr();
1093 const Type* sig_type = TypeOopPtr::make_from_klass(t->as_klass());
1094 if (arg_type != NULL && !arg_type->higher_equal(sig_type)) {
1095 const Type* narrowed_arg_type = arg_type->filter_speculative(sig_type); // keep speculative part
1096 Node* cast_obj = gvn.transform(new CheckCastPPNode(kit.control(), arg, narrowed_arg_type));
1097 kit.set_argument(receiver_skip + j, cast_obj);
1098 }
1099 }
1100 j += t->size(); // long and double take two slots
1101 }
1102
1103 // Try to get the most accurate receiver type
1104 const bool is_virtual = (iid == vmIntrinsics::_linkToVirtual);
1105 const bool is_virtual_or_interface = (is_virtual || iid == vmIntrinsics::_linkToInterface);
1106 int vtable_index = Method::invalid_vtable_index;
1107 bool call_does_dispatch = false;
1108
1109 ciKlass* speculative_receiver_type = NULL;
1110 if (is_virtual_or_interface) {
1111 ciInstanceKlass* klass = target->holder();
1112 Node* receiver_node = kit.argument(0);
1113 const TypeOopPtr* receiver_type = gvn.type(receiver_node)->isa_oopptr();
1114 // call_does_dispatch and vtable_index are out-parameters. They might be changed.
1115 // optimize_virtual_call() takes 2 different holder
1116 // arguments for a corner case that doesn't apply here (see
1117 // Parse::do_call())
1118 target = C->optimize_virtual_call(caller, klass, klass,
1119 target, receiver_type, is_virtual,
1120 call_does_dispatch, vtable_index, // out-parameters
1121 false /* check_access */);
1122 // We lack profiling at this call but type speculation may
1123 // provide us with a type
1124 speculative_receiver_type = (receiver_type != NULL) ? receiver_type->speculative_type() : NULL;
1125 }
1126 CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms,
1127 allow_inline,
1128 PROB_ALWAYS,
1129 speculative_receiver_type);
1130 return cg;
1131 } else {
1132 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1133 "member_name not constant");
1134 }
1135 }
1136 break;
1137
1138 case vmIntrinsics::_linkToNative:
1139 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1140 "native call");
1141 break;
1142
1143 default:
1144 fatal("unexpected intrinsic %d: %s", vmIntrinsics::as_int(iid), vmIntrinsics::name_at(iid));
1145 break;
1146 }
1147 return NULL;
1148 }
1149
1184 // do_intrinsic(0)
1185 // else
1186 // if (predicate(1))
1187 // do_intrinsic(1)
1188 // ...
1189 // else
1190 // do_java_comp
1191
1192 GraphKit kit(jvms);
1193 PhaseGVN& gvn = kit.gvn();
1194
1195 CompileLog* log = kit.C->log();
1196 if (log != NULL) {
1197 log->elem("predicated_intrinsic bci='%d' method='%d'",
1198 jvms->bci(), log->identify(method()));
1199 }
1200
1201 if (!method()->is_static()) {
1202 // We need an explicit receiver null_check before checking its type in predicate.
1203 // We share a map with the caller, so his JVMS gets adjusted.
1204 Node* receiver = kit.null_check_receiver_before_call(method());
1205 if (kit.stopped()) {
1206 return kit.transfer_exceptions_into_jvms();
1207 }
1208 }
1209
1210 int n_predicates = _intrinsic->predicates_count();
1211 assert(n_predicates > 0, "sanity");
1212
1213 JVMState** result_jvms = NEW_RESOURCE_ARRAY(JVMState*, (n_predicates+1));
1214
1215 // Region for normal compilation code if intrinsic failed.
1216 Node* slow_region = new RegionNode(1);
1217
1218 int results = 0;
1219 for (int predicate = 0; (predicate < n_predicates) && !kit.stopped(); predicate++) {
1220 #ifdef ASSERT
1221 JVMState* old_jvms = kit.jvms();
1222 SafePointNode* old_map = kit.map();
1223 Node* old_io = old_map->i_o();
1224 Node* old_mem = old_map->memory();
|
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/bcEscapeAnalyzer.hpp"
27 #include "ci/ciCallSite.hpp"
28 #include "ci/ciObjArray.hpp"
29 #include "ci/ciMemberName.hpp"
30 #include "ci/ciMethodHandle.hpp"
31 #include "classfile/javaClasses.hpp"
32 #include "compiler/compileLog.hpp"
33 #include "opto/addnode.hpp"
34 #include "opto/callGenerator.hpp"
35 #include "opto/callnode.hpp"
36 #include "opto/castnode.hpp"
37 #include "opto/cfgnode.hpp"
38 #include "opto/inlinetypenode.hpp"
39 #include "opto/parse.hpp"
40 #include "opto/rootnode.hpp"
41 #include "opto/runtime.hpp"
42 #include "opto/subnode.hpp"
43 #include "runtime/os.inline.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "utilities/debug.hpp"
46
47 // Utility function.
48 const TypeFunc* CallGenerator::tf() const {
49 return TypeFunc::make(method());
50 }
51
52 bool CallGenerator::is_inlined_method_handle_intrinsic(JVMState* jvms, ciMethod* m) {
53 return is_inlined_method_handle_intrinsic(jvms->method(), jvms->bci(), m);
54 }
55
56 bool CallGenerator::is_inlined_method_handle_intrinsic(ciMethod* caller, int bci, ciMethod* m) {
57 ciMethod* symbolic_info = caller->get_method_at_bci(bci);
58 return is_inlined_method_handle_intrinsic(symbolic_info, m);
102 GraphKit& exits = parser.exits();
103
104 if (C->failing()) {
105 while (exits.pop_exception_state() != NULL) ;
106 return NULL;
107 }
108
109 assert(exits.jvms()->same_calls_as(jvms), "sanity");
110
111 // Simply return the exit state of the parser,
112 // augmented by any exceptional states.
113 return exits.transfer_exceptions_into_jvms();
114 }
115
116 //---------------------------DirectCallGenerator------------------------------
117 // Internal class which handles all out-of-line calls w/o receiver type checks.
118 class DirectCallGenerator : public CallGenerator {
119 private:
120 CallStaticJavaNode* _call_node;
121 // Force separate memory and I/O projections for the exceptional
122 // paths to facilitate late inlining.
123 bool _separate_io_proj;
124
125 protected:
126 void set_call_node(CallStaticJavaNode* call) { _call_node = call; }
127
128 public:
129 DirectCallGenerator(ciMethod* method, bool separate_io_proj)
130 : CallGenerator(method),
131 _call_node(NULL),
132 _separate_io_proj(separate_io_proj)
133 {
134 if (InlineTypeReturnedAsFields && method->is_method_handle_intrinsic()) {
135 // If that call has not been optimized by the time optimizations are over,
136 // we'll need to add a call to create an inline type instance from the klass
137 // returned by the call (see PhaseMacroExpand::expand_mh_intrinsic_return).
138 // Separating memory and I/O projections for exceptions is required to
139 // perform that graph transformation.
140 _separate_io_proj = true;
141 }
142 }
143 virtual JVMState* generate(JVMState* jvms);
144
145 virtual CallNode* call_node() const { return _call_node; }
146 virtual CallGenerator* with_call_node(CallNode* call) {
147 DirectCallGenerator* dcg = new DirectCallGenerator(method(), _separate_io_proj);
148 dcg->set_call_node(call->as_CallStaticJava());
149 return dcg;
150 }
151 };
152
153 JVMState* DirectCallGenerator::generate(JVMState* jvms) {
154 GraphKit kit(jvms);
155 kit.C->print_inlining_update(this);
156 PhaseGVN& gvn = kit.gvn();
157 bool is_static = method()->is_static();
158 address target = is_static ? SharedRuntime::get_resolve_static_call_stub()
159 : SharedRuntime::get_resolve_opt_virtual_call_stub();
160
161 if (kit.C->log() != NULL) {
162 kit.C->log()->elem("direct_call bci='%d'", jvms->bci());
163 }
164
165 CallStaticJavaNode* call = new CallStaticJavaNode(kit.C, tf(), target, method());
166 if (is_inlined_method_handle_intrinsic(jvms, method())) {
167 // To be able to issue a direct call and skip a call to MH.linkTo*/invokeBasic adapter,
168 // additional information about the method being invoked should be attached
169 // to the call site to make resolution logic work
170 // (see SharedRuntime::resolve_static_call_C).
171 call->set_override_symbolic_info(true);
172 }
173 _call_node = call; // Save the call node in case we need it later
174 if (!is_static) {
175 // Make an explicit receiver null_check as part of this call.
176 // Since we share a map with the caller, his JVMS gets adjusted.
177 kit.null_check_receiver_before_call(method());
178 if (kit.stopped()) {
179 // And dump it back to the caller, decorated with any exceptions:
180 return kit.transfer_exceptions_into_jvms();
181 }
182 // Mark the call node as virtual, sort of:
183 call->set_optimized_virtual(true);
184 if (method()->is_method_handle_intrinsic() ||
185 method()->is_compiled_lambda_form()) {
186 call->set_method_handle_invoke(true);
187 }
188 }
189 kit.set_arguments_for_java_call(call, is_late_inline());
190 if (kit.stopped()) {
191 return kit.transfer_exceptions_into_jvms();
192 }
193 kit.set_edges_for_java_call(call, false, _separate_io_proj);
194 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
195 kit.push_node(method()->return_type()->basic_type(), ret);
196 return kit.transfer_exceptions_into_jvms();
197 }
198
199 //--------------------------VirtualCallGenerator------------------------------
200 // Internal class which handles all out-of-line calls checking receiver type.
201 class VirtualCallGenerator : public CallGenerator {
202 private:
203 int _vtable_index;
204 bool _separate_io_proj;
205 CallDynamicJavaNode* _call_node;
206
207 protected:
208 void set_call_node(CallDynamicJavaNode* call) { _call_node = call; }
209
210 public:
211 VirtualCallGenerator(ciMethod* method, int vtable_index, bool separate_io_proj)
212 : CallGenerator(method), _vtable_index(vtable_index), _separate_io_proj(separate_io_proj), _call_node(NULL)
213 {
214 assert(vtable_index == Method::invalid_vtable_index ||
215 vtable_index >= 0, "either invalid or usable");
216 }
217 virtual bool is_virtual() const { return true; }
218 virtual JVMState* generate(JVMState* jvms);
219
220 virtual CallNode* call_node() const { return _call_node; }
221 int vtable_index() const { return _vtable_index; }
222
223 virtual CallGenerator* with_call_node(CallNode* call) {
224 VirtualCallGenerator* cg = new VirtualCallGenerator(method(), _vtable_index, _separate_io_proj);
225 cg->set_call_node(call->as_CallDynamicJava());
226 return cg;
227 }
228 };
229
230 JVMState* VirtualCallGenerator::generate(JVMState* jvms) {
231 GraphKit kit(jvms);
232 Node* receiver = kit.argument(0);
233 kit.C->print_inlining_update(this);
234
235 if (kit.C->log() != NULL) {
236 kit.C->log()->elem("virtual_call bci='%d'", jvms->bci());
237 }
238
239 // If the receiver is a constant null, do not torture the system
240 // by attempting to call through it. The compile will proceed
241 // correctly, but may bail out in final_graph_reshaping, because
242 // the call instruction will have a seemingly deficient out-count.
243 // (The bailout says something misleading about an "infinite loop".)
244 if (!receiver->is_InlineType() && kit.gvn().type(receiver)->higher_equal(TypePtr::NULL_PTR)) {
245 assert(Bytecodes::is_invoke(kit.java_bc()), "%d: %s", kit.java_bc(), Bytecodes::name(kit.java_bc()));
246 ciMethod* declared_method = kit.method()->get_method_at_bci(kit.bci());
247 int arg_size = declared_method->signature()->arg_size_for_bc(kit.java_bc());
248 kit.inc_sp(arg_size); // restore arguments
249 kit.uncommon_trap(Deoptimization::Reason_null_check,
250 Deoptimization::Action_none,
251 NULL, "null receiver");
252 return kit.transfer_exceptions_into_jvms();
253 }
254
255 // Ideally we would unconditionally do a null check here and let it
256 // be converted to an implicit check based on profile information.
257 // However currently the conversion to implicit null checks in
258 // Block::implicit_null_check() only looks for loads and stores, not calls.
259 ciMethod *caller = kit.method();
260 ciMethodData *caller_md = (caller == NULL) ? NULL : caller->method_data();
261 if (!UseInlineCaches || !ImplicitNullChecks || !os::zero_page_read_protected() ||
262 ((ImplicitNullCheckThreshold > 0) && caller_md &&
263 (caller_md->trap_count(Deoptimization::Reason_null_check)
264 >= (uint)ImplicitNullCheckThreshold))) {
272 }
273
274 assert(!method()->is_static(), "virtual call must not be to static");
275 assert(!method()->is_final(), "virtual call should not be to final");
276 assert(!method()->is_private(), "virtual call should not be to private");
277 assert(_vtable_index == Method::invalid_vtable_index || !UseInlineCaches,
278 "no vtable calls if +UseInlineCaches ");
279 address target = SharedRuntime::get_resolve_virtual_call_stub();
280 // Normal inline cache used for call
281 CallDynamicJavaNode* call = new CallDynamicJavaNode(tf(), target, method(), _vtable_index);
282 if (is_inlined_method_handle_intrinsic(jvms, method())) {
283 // To be able to issue a direct call (optimized virtual or virtual)
284 // and skip a call to MH.linkTo*/invokeBasic adapter, additional information
285 // about the method being invoked should be attached to the call site to
286 // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C).
287 call->set_override_symbolic_info(true);
288 }
289 _call_node = call; // Save the call node in case we need it later
290
291 kit.set_arguments_for_java_call(call);
292 if (kit.stopped()) {
293 return kit.transfer_exceptions_into_jvms();
294 }
295 kit.set_edges_for_java_call(call, false /*must_throw*/, _separate_io_proj);
296 Node* ret = kit.set_results_for_java_call(call, _separate_io_proj);
297 kit.push_node(method()->return_type()->basic_type(), ret);
298
299 // Represent the effect of an implicit receiver null_check
300 // as part of this call. Since we share a map with the caller,
301 // his JVMS gets adjusted.
302 kit.cast_not_null(receiver);
303 return kit.transfer_exceptions_into_jvms();
304 }
305
306 CallGenerator* CallGenerator::for_inline(ciMethod* m, float expected_uses) {
307 if (InlineTree::check_can_parse(m) != NULL) return NULL;
308 return new ParseGenerator(m, expected_uses);
309 }
310
311 // As a special case, the JVMS passed to this CallGenerator is
312 // for the method execution already in progress, not just the JVMS
313 // of the caller. Thus, this CallGenerator cannot be mixed with others!
314 CallGenerator* CallGenerator::for_osr(ciMethod* m, int osr_bci) {
368 return DirectCallGenerator::generate(jvms);
369 }
370
371 virtual void print_inlining_late(const char* msg) {
372 CallNode* call = call_node();
373 Compile* C = Compile::current();
374 C->print_inlining_assert_ready();
375 C->print_inlining(method(), call->jvms()->depth()-1, call->jvms()->bci(), msg);
376 C->print_inlining_move_to(this);
377 C->print_inlining_update_delayed(this);
378 }
379
380 virtual void set_unique_id(jlong id) {
381 _unique_id = id;
382 }
383
384 virtual jlong unique_id() const {
385 return _unique_id;
386 }
387
388 virtual CallGenerator* inline_cg() {
389 return _inline_cg;
390 }
391
392 virtual CallGenerator* with_call_node(CallNode* call) {
393 LateInlineCallGenerator* cg = new LateInlineCallGenerator(method(), _inline_cg, _is_pure_call);
394 cg->set_call_node(call->as_CallStaticJava());
395 return cg;
396 }
397 };
398
399 CallGenerator* CallGenerator::for_late_inline(ciMethod* method, CallGenerator* inline_cg) {
400 return new LateInlineCallGenerator(method, inline_cg);
401 }
402
403 class LateInlineMHCallGenerator : public LateInlineCallGenerator {
404 ciMethod* _caller;
405 bool _input_not_const;
406
407 virtual bool do_late_inline_check(Compile* C, JVMState* jvms);
408
409 public:
410 LateInlineMHCallGenerator(ciMethod* caller, ciMethod* callee, bool input_not_const) :
411 LateInlineCallGenerator(callee, NULL), _caller(caller), _input_not_const(input_not_const) {}
433 cg->set_call_node(call->as_CallStaticJava());
434 return cg;
435 }
436 };
437
438 bool LateInlineMHCallGenerator::do_late_inline_check(Compile* C, JVMState* jvms) {
439 // When inlining a virtual call, the null check at the call and the call itself can throw. These 2 paths have different
440 // expression stacks which causes late inlining to break. The MH invoker is not expected to be called from a method with
441 // exception handlers. When there is no exception handler, GraphKit::builtin_throw() pops the stack which solves the issue
442 // of late inlining with exceptions.
443 assert(!jvms->method()->has_exception_handlers() ||
444 (method()->intrinsic_id() != vmIntrinsics::_linkToVirtual &&
445 method()->intrinsic_id() != vmIntrinsics::_linkToInterface), "no exception handler expected");
446 // Even if inlining is not allowed, a virtual call can be strength-reduced to a direct call.
447 bool allow_inline = C->inlining_incrementally();
448 bool input_not_const = true;
449 CallGenerator* cg = for_method_handle_inline(jvms, _caller, method(), allow_inline, input_not_const);
450 assert(!input_not_const, "sanity"); // shouldn't have been scheduled for inlining in the first place
451
452 if (cg != NULL) {
453 // AlwaysIncrementalInline causes for_method_handle_inline() to
454 // return a LateInlineCallGenerator. Extract the
455 // InlineCallGenerator from it.
456 if (AlwaysIncrementalInline && cg->is_late_inline() && !cg->is_virtual_late_inline()) {
457 cg = cg->inline_cg();
458 assert(cg != NULL, "inline call generator expected");
459 }
460
461 assert(!cg->is_late_inline() || cg->is_mh_late_inline() || AlwaysIncrementalInline, "we're doing late inlining");
462 _inline_cg = cg;
463 C->dec_number_of_mh_late_inlines();
464 return true;
465 } else {
466 // Method handle call which has a constant appendix argument should be either inlined or replaced with a direct call
467 // unless there's a signature mismatch between caller and callee. If the failure occurs, there's not much to be improved later,
468 // so don't reinstall the generator to avoid pushing the generator between IGVN and incremental inlining indefinitely.
469 return false;
470 }
471 }
472
473 CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const) {
474 assert(IncrementalInlineMH, "required");
475 Compile::current()->inc_number_of_mh_late_inlines();
476 CallGenerator* cg = new LateInlineMHCallGenerator(caller, callee, input_not_const);
477 return cg;
478 }
479
480 // Allow inlining decisions to be delayed
603
604 void LateInlineMHCallGenerator::do_late_inline() {
605 CallGenerator::do_late_inline_helper();
606 }
607
608 void LateInlineVirtualCallGenerator::do_late_inline() {
609 assert(_callee != NULL, "required"); // set up in CallDynamicJavaNode::Ideal
610 CallGenerator::do_late_inline_helper();
611 }
612
613 void CallGenerator::do_late_inline_helper() {
614 assert(is_late_inline(), "only late inline allowed");
615
616 // Can't inline it
617 CallNode* call = call_node();
618 if (call == NULL || call->outcnt() == 0 ||
619 call->in(0) == NULL || call->in(0)->is_top()) {
620 return;
621 }
622
623 const TypeTuple* r = call->tf()->domain_cc();
624 for (uint i1 = TypeFunc::Parms; i1 < r->cnt(); i1++) {
625 if (call->in(i1)->is_top() && r->field_at(i1) != Type::HALF) {
626 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
627 return;
628 }
629 }
630
631 if (call->in(TypeFunc::Memory)->is_top()) {
632 assert(Compile::current()->inlining_incrementally(), "shouldn't happen during parsing");
633 return;
634 }
635 if (call->in(TypeFunc::Memory)->is_MergeMem()) {
636 MergeMemNode* merge_mem = call->in(TypeFunc::Memory)->as_MergeMem();
637 if (merge_mem->base_memory() == merge_mem->empty_memory()) {
638 return; // dead path
639 }
640 }
641
642 // check for unreachable loop
643 CallProjections* callprojs = call->extract_projections(true);
644 if ((callprojs->fallthrough_catchproj == call->in(0)) ||
645 (callprojs->catchall_catchproj == call->in(0)) ||
646 (callprojs->fallthrough_memproj == call->in(TypeFunc::Memory)) ||
647 (callprojs->catchall_memproj == call->in(TypeFunc::Memory)) ||
648 (callprojs->fallthrough_ioproj == call->in(TypeFunc::I_O)) ||
649 (callprojs->catchall_ioproj == call->in(TypeFunc::I_O)) ||
650 (callprojs->exobj != NULL && call->find_edge(callprojs->exobj) != -1)) {
651 return;
652 }
653
654 Compile* C = Compile::current();
655 // Remove inlined methods from Compiler's lists.
656 if (call->is_macro()) {
657 C->remove_macro_node(call);
658 }
659
660
661 bool result_not_used = true;
662 for (uint i = 0; i < callprojs->nb_resproj; i++) {
663 if (callprojs->resproj[i] != NULL) {
664 if (callprojs->resproj[i]->outcnt() != 0) {
665 result_not_used = false;
666 }
667 if (call->find_edge(callprojs->resproj[i]) != -1) {
668 return;
669 }
670 }
671 }
672
673 if (is_pure_call() && result_not_used) {
674 // The call is marked as pure (no important side effects), but result isn't used.
675 // It's safe to remove the call.
676 GraphKit kit(call->jvms());
677 kit.replace_call(call, C->top(), true);
678 } else {
679 // Make a clone of the JVMState that appropriate to use for driving a parse
680 JVMState* old_jvms = call->jvms();
681 JVMState* jvms = old_jvms->clone_shallow(C);
682 uint size = call->req();
683 SafePointNode* map = new SafePointNode(size, jvms);
684 for (uint i1 = 0; i1 < size; i1++) {
685 map->init_req(i1, call->in(i1));
686 }
687
688 PhaseGVN& gvn = *C->initial_gvn();
689 // Make sure the state is a MergeMem for parsing.
690 if (!map->in(TypeFunc::Memory)->is_MergeMem()) {
691 Node* mem = MergeMemNode::make(map->in(TypeFunc::Memory));
692 gvn.set_type_bottom(mem);
693 map->set_req(TypeFunc::Memory, mem);
694 }
695
696 // blow away old call arguments
697 for (uint i1 = TypeFunc::Parms; i1 < r->cnt(); i1++) {
698 map->set_req(i1, C->top());
699 }
700 jvms->set_map(map);
701
702 // Make enough space in the expression stack to transfer
703 // the incoming arguments and return value.
704 map->ensure_stack(jvms, jvms->method()->max_stack());
705 const TypeTuple* domain_sig = call->_tf->domain_sig();
706 uint nargs = method()->arg_size();
707 assert(domain_sig->cnt() - TypeFunc::Parms == nargs, "inconsistent signature");
708
709 uint j = TypeFunc::Parms;
710 int arg_num = 0;
711 for (uint i1 = 0; i1 < nargs; i1++) {
712 const Type* t = domain_sig->field_at(TypeFunc::Parms + i1);
713 if (t->is_inlinetypeptr() && method()->is_scalarized_arg(arg_num)) {
714 // Inline type arguments are not passed by reference: we get an argument per
715 // field of the inline type. Build InlineTypeNodes from the inline type arguments.
716 GraphKit arg_kit(jvms, &gvn);
717 Node* vt = InlineTypeNode::make_from_multi(&arg_kit, call, t->inline_klass(), j, /* in= */ true, /* null_free= */ !t->maybe_null());
718 map->set_control(arg_kit.control());
719 map->set_argument(jvms, i1, vt);
720 } else {
721 map->set_argument(jvms, i1, call->in(j++));
722 }
723 if (t != Type::HALF) {
724 arg_num++;
725 }
726 }
727
728 C->print_inlining_assert_ready();
729
730 C->print_inlining_move_to(this);
731
732 C->log_late_inline(this);
733
734 // JVMState is ready, so time to perform some checks and prepare for inlining attempt.
735 if (!do_late_inline_check(C, jvms)) {
736 map->disconnect_inputs(C);
737 C->print_inlining_update_delayed(this);
738 return;
739 }
740
741 // Check if we are late inlining a method handle call that returns an inline type as fields.
742 Node* buffer_oop = NULL;
743 ciMethod* inline_method = inline_cg()->method();
744 ciType* return_type = inline_method->return_type();
745 if (!call->tf()->returns_inline_type_as_fields() && is_mh_late_inline() &&
746 return_type->is_inlinetype() && return_type->as_inline_klass()->can_be_returned_as_fields()) {
747 // Allocate a buffer for the inline type returned as fields because the caller expects an oop return.
748 // Do this before the method handle call in case the buffer allocation triggers deoptimization and
749 // we need to "re-execute" the call in the interpreter (to make sure the call is only executed once).
750 GraphKit arg_kit(jvms, &gvn);
751 {
752 PreserveReexecuteState preexecs(&arg_kit);
753 arg_kit.jvms()->set_should_reexecute(true);
754 arg_kit.inc_sp(nargs);
755 Node* klass_node = arg_kit.makecon(TypeKlassPtr::make(return_type->as_inline_klass()));
756 buffer_oop = arg_kit.new_instance(klass_node, NULL, NULL, /* deoptimize_on_exception */ true);
757 }
758 jvms = arg_kit.transfer_exceptions_into_jvms();
759 }
760
761 // Setup default node notes to be picked up by the inlining
762 Node_Notes* old_nn = C->node_notes_at(call->_idx);
763 if (old_nn != NULL) {
764 Node_Notes* entry_nn = old_nn->clone(C);
765 entry_nn->set_jvms(jvms);
766 C->set_default_node_notes(entry_nn);
767 }
768
769 // Now perform the inlining using the synthesized JVMState
770 JVMState* new_jvms = inline_cg()->generate(jvms);
771 if (new_jvms == NULL) return; // no change
772 if (C->failing()) return;
773
774 // Capture any exceptional control flow
775 GraphKit kit(new_jvms);
776
777 // Find the result object
778 Node* result = C->top();
779 int result_size = method()->return_type()->size();
780 if (result_size != 0 && !kit.stopped()) {
781 result = (result_size == 1) ? kit.pop() : kit.pop_pair();
782 }
783
784 if (inline_cg()->is_inline()) {
785 C->set_has_loops(C->has_loops() || inline_method->has_loops());
786 C->env()->notice_inlined_method(inline_method);
787 }
788 C->set_inlining_progress(true);
789 C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup
790
791 // Handle inline type returns
792 InlineTypeBaseNode* vt = result->isa_InlineTypeBase();
793 if (vt != NULL) {
794 if (call->tf()->returns_inline_type_as_fields()) {
795 vt->replace_call_results(&kit, call, C, inline_method->signature()->returns_null_free_inline_type());
796 } else if (vt->is_InlineType()) {
797 // Result might still be allocated (for example, if it has been stored to a non-flattened field)
798 if (!vt->is_allocated(&kit.gvn())) {
799 assert(buffer_oop != NULL, "should have allocated a buffer");
800 RegionNode* region = new RegionNode(3);
801
802 // Check if result is null
803 Node* null_ctl = kit.top();
804 if (!inline_method->signature()->returns_null_free_inline_type()) {
805 kit.null_check_common(vt->get_is_init(), T_INT, false, &null_ctl);
806 }
807 region->init_req(1, null_ctl);
808 PhiNode* oop = PhiNode::make(region, kit.gvn().zerocon(T_OBJECT), TypeInstPtr::make(TypePtr::BotPTR, vt->type()->inline_klass()));
809 Node* init_mem = kit.reset_memory();
810 PhiNode* mem = PhiNode::make(region, init_mem, Type::MEMORY, TypePtr::BOTTOM);
811
812 // Not null, initialize the buffer
813 kit.set_all_memory(init_mem);
814 vt->store(&kit, buffer_oop, buffer_oop, vt->type()->inline_klass());
815 // Do not let stores that initialize this buffer be reordered with a subsequent
816 // store that would make this buffer accessible by other threads.
817 AllocateNode* alloc = AllocateNode::Ideal_allocation(buffer_oop, &kit.gvn());
818 assert(alloc != NULL, "must have an allocation node");
819 kit.insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress));
820 region->init_req(2, kit.control());
821 oop->init_req(2, buffer_oop);
822 mem->init_req(2, kit.merged_memory());
823
824 // Update oop input to buffer
825 kit.gvn().hash_delete(vt);
826 vt->set_oop(kit.gvn().transform(oop));
827 vt = kit.gvn().transform(vt)->as_InlineTypeBase();
828
829 kit.set_control(kit.gvn().transform(region));
830 kit.set_all_memory(kit.gvn().transform(mem));
831 kit.record_for_igvn(region);
832 kit.record_for_igvn(oop);
833 kit.record_for_igvn(mem);
834 }
835 result = vt->as_ptr(&kit.gvn(), inline_method->signature()->returns_null_free_inline_type());
836 }
837 DEBUG_ONLY(buffer_oop = NULL);
838 } else {
839 assert(result->is_top() || !call->tf()->returns_inline_type_as_fields(), "Unexpected return value");
840 }
841 assert(buffer_oop == NULL, "unused buffer allocation");
842
843 kit.replace_call(call, result, true);
844 }
845 }
846
847 class LateInlineStringCallGenerator : public LateInlineCallGenerator {
848
849 public:
850 LateInlineStringCallGenerator(ciMethod* method, CallGenerator* inline_cg) :
851 LateInlineCallGenerator(method, inline_cg) {}
852
853 virtual JVMState* generate(JVMState* jvms) {
854 Compile *C = Compile::current();
855
856 C->log_inline_id(this);
857
858 C->add_string_late_inline(this);
859
860 JVMState* new_jvms = DirectCallGenerator::generate(jvms);
861 return new_jvms;
862 }
1045 // Inline failed, so make a direct call.
1046 assert(_if_hit->is_inline(), "must have been a failed inline");
1047 CallGenerator* cg = CallGenerator::for_direct_call(_if_hit->method());
1048 new_jvms = cg->generate(kit.sync_jvms());
1049 }
1050 kit.add_exception_states_from(new_jvms);
1051 kit.set_jvms(new_jvms);
1052
1053 // Need to merge slow and fast?
1054 if (slow_map == NULL) {
1055 // The fast path is the only path remaining.
1056 return kit.transfer_exceptions_into_jvms();
1057 }
1058
1059 if (kit.stopped()) {
1060 // Inlined method threw an exception, so it's just the slow path after all.
1061 kit.set_jvms(slow_jvms);
1062 return kit.transfer_exceptions_into_jvms();
1063 }
1064
1065 // Allocate inline types if they are merged with objects (similar to Parse::merge_common())
1066 uint tos = kit.jvms()->stkoff() + kit.sp();
1067 uint limit = slow_map->req();
1068 for (uint i = TypeFunc::Parms; i < limit; i++) {
1069 Node* m = kit.map()->in(i);
1070 Node* n = slow_map->in(i);
1071 const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
1072 if (m->is_InlineType() && !t->isa_inlinetype()) {
1073 // Allocate inline type in fast path
1074 m = m->as_InlineType()->buffer(&kit);
1075 kit.map()->set_req(i, m);
1076 }
1077 if (n->is_InlineType() && !t->isa_inlinetype()) {
1078 // Allocate inline type in slow path
1079 PreserveJVMState pjvms(&kit);
1080 kit.set_map(slow_map);
1081 n = n->as_InlineType()->buffer(&kit);
1082 kit.map()->set_req(i, n);
1083 slow_map = kit.stop();
1084 }
1085 }
1086
1087 // There are 2 branches and the replaced nodes are only valid on
1088 // one: restore the replaced nodes to what they were before the
1089 // branch.
1090 kit.map()->set_replaced_nodes(replaced_nodes);
1091
1092 // Finish the diamond.
1093 kit.C->set_has_split_ifs(true); // Has chance for split-if optimization
1094 RegionNode* region = new RegionNode(3);
1095 region->init_req(1, kit.control());
1096 region->init_req(2, slow_map->control());
1097 kit.set_control(gvn.transform(region));
1098 Node* iophi = PhiNode::make(region, kit.i_o(), Type::ABIO);
1099 iophi->set_req(2, slow_map->i_o());
1100 kit.set_i_o(gvn.transform(iophi));
1101 // Merge memory
1102 kit.merge_memory(slow_map->merged_memory(), region, 2);
1103 // Transform new memory Phis.
1104 for (MergeMemStream mms(kit.merged_memory()); mms.next_non_empty();) {
1105 Node* phi = mms.memory();
1106 if (phi->is_Phi() && phi->in(0) == region) {
1107 mms.set_memory(gvn.transform(phi));
1108 }
1109 }
1110 for (uint i = TypeFunc::Parms; i < limit; i++) {
1111 // Skip unused stack slots; fast forward to monoff();
1112 if (i == tos) {
1113 i = kit.jvms()->monoff();
1114 if( i >= limit ) break;
1115 }
1116 Node* m = kit.map()->in(i);
1117 Node* n = slow_map->in(i);
1118 if (m != n) {
1119 const Type* t = gvn.type(m)->meet_speculative(gvn.type(n));
1120 Node* phi = PhiNode::make(region, m, t);
1121 phi->set_req(2, n);
1122 kit.map()->set_req(i, gvn.transform(phi));
1123 }
1124 }
1125 return kit.transfer_exceptions_into_jvms();
1126 }
1127
1128
1129 CallGenerator* CallGenerator::for_method_handle_call(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline) {
1130 assert(callee->is_method_handle_intrinsic(), "for_method_handle_call mismatch");
1131 bool input_not_const;
1132 CallGenerator* cg = CallGenerator::for_method_handle_inline(jvms, caller, callee, allow_inline, input_not_const);
1133 Compile* C = Compile::current();
1134 if (cg != NULL) {
1135 if (AlwaysIncrementalInline) {
1136 return CallGenerator::for_late_inline(callee, cg);
1137 } else {
1138 return cg;
1139 }
1140 }
1141 int bci = jvms->bci();
1142 ciCallProfile profile = caller->call_profile_at_bci(bci);
1143 int call_site_count = caller->scale_count(profile.count());
1144
1145 if (IncrementalInlineMH && (AlwaysIncrementalInline ||
1146 (call_site_count > 0 && (input_not_const || !C->inlining_incrementally() || C->over_inlining_cutoff())))) {
1147 return CallGenerator::for_mh_late_inline(caller, callee, input_not_const);
1148 } else {
1149 // Out-of-line call.
1150 return CallGenerator::for_direct_call(callee);
1151 }
1152 }
1153
1154 static void cast_argument(int nargs, int arg_nb, ciType* t, GraphKit& kit, bool null_free) {
1155 PhaseGVN& gvn = kit.gvn();
1156 Node* arg = kit.argument(arg_nb);
1157 const Type* arg_type = arg->bottom_type();
1158 const Type* sig_type = TypeOopPtr::make_from_klass(t->as_klass());
1159 if (t->as_klass()->is_inlinetype() && null_free) {
1160 sig_type = sig_type->filter_speculative(TypePtr::NOTNULL);
1161 }
1162 if (arg_type->isa_oopptr() && !arg_type->higher_equal(sig_type)) {
1163 const Type* narrowed_arg_type = arg_type->filter_speculative(sig_type); // keep speculative part
1164 arg = gvn.transform(new CheckCastPPNode(kit.control(), arg, narrowed_arg_type));
1165 kit.set_argument(arg_nb, arg);
1166 }
1167 if (sig_type->is_inlinetypeptr() && !arg->is_InlineType()) {
1168 arg = InlineTypeNode::make_from_oop(&kit, arg, t->as_inline_klass(), !kit.gvn().type(arg)->maybe_null());
1169 kit.set_argument(arg_nb, arg);
1170 }
1171 }
1172
1173 CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod* caller, ciMethod* callee, bool allow_inline, bool& input_not_const) {
1174 GraphKit kit(jvms);
1175 PhaseGVN& gvn = kit.gvn();
1176 Compile* C = kit.C;
1177 vmIntrinsics::ID iid = callee->intrinsic_id();
1178 input_not_const = true;
1179 if (StressMethodHandleLinkerInlining) {
1180 allow_inline = false;
1181 }
1182 switch (iid) {
1183 case vmIntrinsics::_invokeBasic:
1184 {
1185 // Get MethodHandle receiver:
1186 Node* receiver = kit.argument(0);
1187 if (receiver->Opcode() == Op_ConP) {
1188 input_not_const = false;
1189 const TypeOopPtr* recv_toop = receiver->bottom_type()->isa_oopptr();
1190 if (recv_toop != NULL) {
1191 ciMethod* target = recv_toop->const_oop()->as_method_handle()->get_vmtarget();
1192 const int vtable_index = Method::invalid_vtable_index;
1204 PROB_ALWAYS);
1205 return cg;
1206 } else {
1207 assert(receiver->bottom_type() == TypePtr::NULL_PTR, "not a null: %s",
1208 Type::str(receiver->bottom_type()));
1209 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1210 "receiver is always null");
1211 }
1212 } else {
1213 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1214 "receiver not constant");
1215 }
1216 }
1217 break;
1218
1219 case vmIntrinsics::_linkToVirtual:
1220 case vmIntrinsics::_linkToStatic:
1221 case vmIntrinsics::_linkToSpecial:
1222 case vmIntrinsics::_linkToInterface:
1223 {
1224 int nargs = callee->arg_size();
1225 // Get MemberName argument:
1226 Node* member_name = kit.argument(nargs - 1);
1227 if (member_name->Opcode() == Op_ConP) {
1228 input_not_const = false;
1229 const TypeOopPtr* oop_ptr = member_name->bottom_type()->is_oopptr();
1230 ciMethod* target = oop_ptr->const_oop()->as_member_name()->get_vmtarget();
1231
1232 if (!ciMethod::is_consistent_info(callee, target)) {
1233 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1234 "signatures mismatch");
1235 return NULL;
1236 }
1237
1238 // In lambda forms we erase signature types to avoid resolving issues
1239 // involving class loaders. When we optimize a method handle invoke
1240 // to a direct call we must cast the receiver and arguments to its
1241 // actual types.
1242 ciSignature* signature = target->signature();
1243 const int receiver_skip = target->is_static() ? 0 : 1;
1244 // Cast receiver to its type.
1245 if (!target->is_static()) {
1246 cast_argument(nargs, 0, signature->accessing_klass(), kit, false);
1247 }
1248 // Cast reference arguments to its type.
1249 for (int i = 0, j = 0; i < signature->count(); i++) {
1250 ciType* t = signature->type_at(i);
1251 if (t->is_klass()) {
1252 bool null_free = signature->is_null_free_at(i);
1253 cast_argument(nargs, receiver_skip + j, t, kit, null_free);
1254 }
1255 j += t->size(); // long and double take two slots
1256 }
1257
1258 // Try to get the most accurate receiver type
1259 const bool is_virtual = (iid == vmIntrinsics::_linkToVirtual);
1260 const bool is_virtual_or_interface = (is_virtual || iid == vmIntrinsics::_linkToInterface);
1261 int vtable_index = Method::invalid_vtable_index;
1262 bool call_does_dispatch = false;
1263
1264 ciKlass* speculative_receiver_type = NULL;
1265 if (is_virtual_or_interface) {
1266 ciInstanceKlass* klass = target->holder();
1267 Node* receiver_node = kit.argument(0);
1268 const TypeOopPtr* receiver_type = gvn.type(receiver_node)->isa_oopptr();
1269 // call_does_dispatch and vtable_index are out-parameters. They might be changed.
1270 // optimize_virtual_call() takes 2 different holder
1271 // arguments for a corner case that doesn't apply here (see
1272 // Parse::do_call())
1273 target = C->optimize_virtual_call(caller, klass, klass,
1274 target, receiver_type, is_virtual,
1275 call_does_dispatch, vtable_index, // out-parameters
1276 false /* check_access */);
1277 // We lack profiling at this call but type speculation may
1278 // provide us with a type
1279 speculative_receiver_type = (receiver_type != NULL) ? receiver_type->speculative_type() : NULL;
1280 }
1281 CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms,
1282 allow_inline,
1283 PROB_ALWAYS,
1284 speculative_receiver_type,
1285 true);
1286 return cg;
1287 } else {
1288 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1289 "member_name not constant");
1290 }
1291 }
1292 break;
1293
1294 case vmIntrinsics::_linkToNative:
1295 print_inlining_failure(C, callee, jvms->depth() - 1, jvms->bci(),
1296 "native call");
1297 break;
1298
1299 default:
1300 fatal("unexpected intrinsic %d: %s", vmIntrinsics::as_int(iid), vmIntrinsics::name_at(iid));
1301 break;
1302 }
1303 return NULL;
1304 }
1305
1340 // do_intrinsic(0)
1341 // else
1342 // if (predicate(1))
1343 // do_intrinsic(1)
1344 // ...
1345 // else
1346 // do_java_comp
1347
1348 GraphKit kit(jvms);
1349 PhaseGVN& gvn = kit.gvn();
1350
1351 CompileLog* log = kit.C->log();
1352 if (log != NULL) {
1353 log->elem("predicated_intrinsic bci='%d' method='%d'",
1354 jvms->bci(), log->identify(method()));
1355 }
1356
1357 if (!method()->is_static()) {
1358 // We need an explicit receiver null_check before checking its type in predicate.
1359 // We share a map with the caller, so his JVMS gets adjusted.
1360 kit.null_check_receiver_before_call(method());
1361 if (kit.stopped()) {
1362 return kit.transfer_exceptions_into_jvms();
1363 }
1364 }
1365
1366 int n_predicates = _intrinsic->predicates_count();
1367 assert(n_predicates > 0, "sanity");
1368
1369 JVMState** result_jvms = NEW_RESOURCE_ARRAY(JVMState*, (n_predicates+1));
1370
1371 // Region for normal compilation code if intrinsic failed.
1372 Node* slow_region = new RegionNode(1);
1373
1374 int results = 0;
1375 for (int predicate = 0; (predicate < n_predicates) && !kit.stopped(); predicate++) {
1376 #ifdef ASSERT
1377 JVMState* old_jvms = kit.jvms();
1378 SafePointNode* old_map = kit.map();
1379 Node* old_io = old_map->i_o();
1380 Node* old_mem = old_map->memory();
|