16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/barrierSet.hpp"
27 #include "gc/shared/c2/barrierSetC2.hpp"
28 #include "memory/allocation.inline.hpp"
29 #include "memory/resourceArea.hpp"
30 #include "opto/addnode.hpp"
31 #include "opto/callnode.hpp"
32 #include "opto/castnode.hpp"
33 #include "opto/connode.hpp"
34 #include "opto/castnode.hpp"
35 #include "opto/divnode.hpp"
36 #include "opto/loopnode.hpp"
37 #include "opto/matcher.hpp"
38 #include "opto/mulnode.hpp"
39 #include "opto/movenode.hpp"
40 #include "opto/opaquenode.hpp"
41 #include "opto/rootnode.hpp"
42 #include "opto/subnode.hpp"
43 #include "opto/subtypenode.hpp"
44 #include "opto/superword.hpp"
45 #include "opto/vectornode.hpp"
46 #include "utilities/macros.hpp"
47
48 //=============================================================================
49 //------------------------------split_thru_phi---------------------------------
50 // Split Node 'n' through merge point if there is enough win.
51 Node* PhaseIdealLoop::split_thru_phi(Node* n, Node* region, int policy) {
52 if ((n->Opcode() == Op_ConvI2L && n->bottom_type() != TypeLong::LONG) ||
53 (n->Opcode() == Op_ConvL2I && n->bottom_type() != TypeInt::INT)) {
54 // ConvI2L/ConvL2I may have type information on it which is unsafe to push up
55 // so disable this for now
56 return nullptr;
57 }
58
59 // Splitting range check CastIIs through a loop induction Phi can
60 // cause new Phis to be created that are left unrelated to the loop
61 // induction Phi and prevent optimizations (vectorization)
62 if (n->Opcode() == Op_CastII && region->is_CountedLoop() &&
63 n->in(1) == region->as_CountedLoop()->phi()) {
64 return nullptr;
65 }
66
67 if (cannot_split_division(n, region)) {
68 return nullptr;
69 }
70
71 int wins = 0;
72 assert(!n->is_CFG(), "");
73 assert(region->is_Region(), "");
74
75 const Type* type = n->bottom_type();
76 const TypeOopPtr* t_oop = _igvn.type(n)->isa_oopptr();
77 Node* phi;
78 if (t_oop != nullptr && t_oop->is_known_instance_field()) {
79 int iid = t_oop->instance_id();
80 int index = C->get_alias_index(t_oop);
81 int offset = t_oop->offset();
82 phi = new PhiNode(region, type, nullptr, iid, index, offset);
83 } else {
84 phi = PhiNode::make_blank(region, n);
85 }
86 uint old_unique = C->unique();
743 // CMOVE'd derived pointer? It's a CMOVE'd derived base. Thus
744 // CMOVE'ing a derived pointer requires we also CMOVE the base. If we
745 // have a Phi for the base here that we convert to a CMOVE all is well
746 // and good. But if the base is dead, we'll not make a CMOVE. Later
747 // the allocator will have to produce a base by creating a CMOVE of the
748 // relevant bases. This puts the allocator in the business of
749 // manufacturing expensive instructions, generally a bad plan.
750 // Just Say No to Conditionally-Moved Derived Pointers.
751 if (tp && tp->offset() != 0)
752 return nullptr;
753 cost++;
754 break;
755 }
756 default:
757 return nullptr; // In particular, can't do memory or I/O
758 }
759 // Add in cost any speculative ops
760 for (uint j = 1; j < region->req(); j++) {
761 Node *proj = region->in(j);
762 Node *inp = phi->in(j);
763 if (get_ctrl(inp) == proj) { // Found local op
764 cost++;
765 // Check for a chain of dependent ops; these will all become
766 // speculative in a CMOV.
767 for (uint k = 1; k < inp->req(); k++)
768 if (get_ctrl(inp->in(k)) == proj)
769 cost += ConditionalMoveLimit; // Too much speculative goo
770 }
771 }
772 // See if the Phi is used by a Cmp or Narrow oop Decode/Encode.
773 // This will likely Split-If, a higher-payoff operation.
774 for (DUIterator_Fast kmax, k = phi->fast_outs(kmax); k < kmax; k++) {
775 Node* use = phi->fast_out(k);
776 if (use->is_Cmp() || use->is_DecodeNarrowPtr() || use->is_EncodeNarrowPtr())
777 cost += ConditionalMoveLimit;
778 // Is there a use inside the loop?
779 // Note: check only basic types since CMoveP is pinned.
780 if (!used_inside_loop && is_java_primitive(bt)) {
781 IdealLoopTree* u_loop = get_loop(has_ctrl(use) ? get_ctrl(use) : use);
782 if (r_loop == u_loop || r_loop->is_member(u_loop)) {
1071 assert(get_loop(lca)->_nest < n_loop->_nest || get_loop(lca)->_head->as_Loop()->is_in_infinite_subgraph(), "must not be moved into inner loop");
1072
1073 // Move store out of the loop
1074 _igvn.replace_node(hook, n->in(MemNode::Memory));
1075 _igvn.replace_input_of(n, 0, lca);
1076 set_ctrl_and_loop(n, lca);
1077
1078 // Disconnect the phi now. An empty phi can confuse other
1079 // optimizations in this pass of loop opts..
1080 if (phi->in(LoopNode::LoopBackControl) == phi) {
1081 _igvn.replace_node(phi, phi->in(LoopNode::EntryControl));
1082 n_loop->_body.yank(phi);
1083 }
1084 }
1085 }
1086 }
1087 }
1088 }
1089 }
1090
1091 // Split some nodes that take a counted loop phi as input at a counted
1092 // loop can cause vectorization of some expressions to fail
1093 bool PhaseIdealLoop::split_thru_phi_could_prevent_vectorization(Node* n, Node* n_blk) {
1094 if (!n_blk->is_CountedLoop()) {
1095 return false;
1096 }
1097
1098 int opcode = n->Opcode();
1099
1100 if (opcode != Op_AndI &&
1101 opcode != Op_MulI &&
1102 opcode != Op_RotateRight &&
1103 opcode != Op_RShiftI) {
1104 return false;
1105 }
1106
1107 return n->in(1) == n_blk->as_BaseCountedLoop()->phi();
1108 }
1109
1110 //------------------------------split_if_with_blocks_pre-----------------------
1111 // Do the real work in a non-recursive function. Data nodes want to be
1112 // cloned in the pre-order so they can feed each other nicely.
1113 Node *PhaseIdealLoop::split_if_with_blocks_pre( Node *n ) {
1114 // Cloning these guys is unlikely to win
1115 int n_op = n->Opcode();
1116 if (n_op == Op_MergeMem) {
1117 return n;
1118 }
1119 if (n->is_Proj()) {
1120 return n;
1121 }
1122 // Do not clone-up CmpFXXX variations, as these are always
1123 // followed by a CmpI
1124 if (n->is_Cmp()) {
1125 return n;
1126 }
1127 // Attempt to use a conditional move instead of a phi/branch
1128 if (ConditionalMoveLimit > 0 && n_op == Op_Region) {
1129 Node *cmov = conditional_move( n );
1130 if (cmov) {
1131 return cmov;
1132 }
1133 }
1134 if (n->is_CFG() || n->is_LoadStore()) {
1135 return n;
1136 }
1137 if (n->is_Opaque1()) { // Opaque nodes cannot be mod'd
1138 if (!C->major_progress()) { // If chance of no more loop opts...
1139 _igvn._worklist.push(n); // maybe we'll remove them
1140 }
1141 return n;
1381
1382 return true;
1383 }
1384
1385 // Detect if the node is the inner strip-mined loop
1386 // Return: null if it's not the case, or the exit of outer strip-mined loop
1387 static Node* is_inner_of_stripmined_loop(const Node* out) {
1388 Node* out_le = nullptr;
1389
1390 if (out->is_CountedLoopEnd()) {
1391 const CountedLoopNode* loop = out->as_CountedLoopEnd()->loopnode();
1392
1393 if (loop != nullptr && loop->is_strip_mined()) {
1394 out_le = loop->in(LoopNode::EntryControl)->as_OuterStripMinedLoop()->outer_loop_exit();
1395 }
1396 }
1397
1398 return out_le;
1399 }
1400
1401 //------------------------------split_if_with_blocks_post----------------------
1402 // Do the real work in a non-recursive function. CFG hackery wants to be
1403 // in the post-order, so it can dirty the I-DOM info and not use the dirtied
1404 // info.
1405 void PhaseIdealLoop::split_if_with_blocks_post(Node *n) {
1406
1407 // Cloning Cmp through Phi's involves the split-if transform.
1408 // FastLock is not used by an If
1409 if (n->is_Cmp() && !n->is_FastLock()) {
1410 Node *n_ctrl = get_ctrl(n);
1411 // Determine if the Node has inputs from some local Phi.
1412 // Returns the block to clone thru.
1413 Node *n_blk = has_local_phi_input(n);
1414 if (n_blk != n_ctrl) {
1415 return;
1416 }
1417
1418 if (!can_split_if(n_ctrl)) {
1419 return;
1420 }
1421
1422 if (n->outcnt() != 1) {
1423 return; // Multiple bool's from 1 compare?
1424 }
1425 Node *bol = n->unique_out();
1426 assert(bol->is_Bool(), "expect a bool here");
1534 // 1. Move from RangeCheck "a" to RangeCheck "b": don't need to pin. If we ever remove b, then we pin
1535 // all its array accesses at that point.
1536 // 2. We move from RangeCheck "a" to regular if "b": need to pin. If we ever remove b, then its array
1537 // accesses would start to float, since we don't pin at that point.
1538 // 3. If we move from regular if: don't pin. All array accesses are already assumed to be pinned.
1539 bool pin_array_access_nodes = n->Opcode() == Op_RangeCheck &&
1540 prevdom->in(0)->Opcode() != Op_RangeCheck;
1541 dominated_by(prevdom->as_IfProj(), n->as_If(), false, pin_array_access_nodes);
1542 DEBUG_ONLY( if (VerifyLoopOptimizations) { verify(); } );
1543 return;
1544 }
1545 prevdom = dom;
1546 dom = idom(prevdom);
1547 }
1548 }
1549 }
1550
1551 try_sink_out_of_loop(n);
1552
1553 try_move_store_after_loop(n);
1554 }
1555
1556 // Transform:
1557 //
1558 // if (some_condition) {
1559 // // body 1
1560 // } else {
1561 // // body 2
1562 // }
1563 // if (some_condition) {
1564 // // body 3
1565 // } else {
1566 // // body 4
1567 // }
1568 //
1569 // into:
1570 //
1571 //
1572 // if (some_condition) {
1573 // // body 1
2018 uint i;
2019 for (i = 1; i < phi->req(); i++) {
2020 Node* b = phi->in(i);
2021 if (b->is_Phi()) {
2022 _igvn.replace_input_of(phi, i, clone_iff(b->as_Phi()));
2023 } else {
2024 assert(b->is_Bool() || b->is_OpaqueNotNull() || b->is_OpaqueInitializedAssertionPredicate(),
2025 "bool, non-null check with OpaqueNotNull or Initialized Assertion Predicate with its Opaque node");
2026 }
2027 }
2028 Node* n = phi->in(1);
2029 Node* sample_opaque = nullptr;
2030 Node *sample_bool = nullptr;
2031 if (n->is_OpaqueNotNull() || n->is_OpaqueInitializedAssertionPredicate()) {
2032 sample_opaque = n;
2033 sample_bool = n->in(1);
2034 assert(sample_bool->is_Bool(), "wrong type");
2035 } else {
2036 sample_bool = n;
2037 }
2038 Node *sample_cmp = sample_bool->in(1);
2039
2040 // Make Phis to merge the Cmp's inputs.
2041 PhiNode *phi1 = new PhiNode(phi->in(0), Type::TOP);
2042 PhiNode *phi2 = new PhiNode(phi->in(0), Type::TOP);
2043 for (i = 1; i < phi->req(); i++) {
2044 Node *n1 = sample_opaque == nullptr ? phi->in(i)->in(1)->in(1) : phi->in(i)->in(1)->in(1)->in(1);
2045 Node *n2 = sample_opaque == nullptr ? phi->in(i)->in(1)->in(2) : phi->in(i)->in(1)->in(1)->in(2);
2046 phi1->set_req(i, n1);
2047 phi2->set_req(i, n2);
2048 phi1->set_type(phi1->type()->meet_speculative(n1->bottom_type()));
2049 phi2->set_type(phi2->type()->meet_speculative(n2->bottom_type()));
2050 }
2051 // See if these Phis have been made before.
2052 // Register with optimizer
2053 Node *hit1 = _igvn.hash_find_insert(phi1);
2054 if (hit1) { // Hit, toss just made Phi
2055 _igvn.remove_dead_node(phi1); // Remove new phi
2056 assert(hit1->is_Phi(), "" );
2057 phi1 = (PhiNode*)hit1; // Use existing phi
2058 } else { // Miss
2059 _igvn.register_new_node_with_optimizer(phi1);
2060 }
2061 Node *hit2 = _igvn.hash_find_insert(phi2);
|
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/barrierSet.hpp"
27 #include "gc/shared/c2/barrierSetC2.hpp"
28 #include "memory/allocation.inline.hpp"
29 #include "memory/resourceArea.hpp"
30 #include "opto/addnode.hpp"
31 #include "opto/callnode.hpp"
32 #include "opto/castnode.hpp"
33 #include "opto/connode.hpp"
34 #include "opto/castnode.hpp"
35 #include "opto/divnode.hpp"
36 #include "opto/inlinetypenode.hpp"
37 #include "opto/loopnode.hpp"
38 #include "opto/matcher.hpp"
39 #include "opto/mulnode.hpp"
40 #include "opto/movenode.hpp"
41 #include "opto/opaquenode.hpp"
42 #include "opto/rootnode.hpp"
43 #include "opto/subnode.hpp"
44 #include "opto/subtypenode.hpp"
45 #include "opto/superword.hpp"
46 #include "opto/vectornode.hpp"
47 #include "utilities/macros.hpp"
48
49 //=============================================================================
50 //------------------------------split_thru_phi---------------------------------
51 // Split Node 'n' through merge point if there is enough win.
52 Node* PhaseIdealLoop::split_thru_phi(Node* n, Node* region, int policy) {
53 if ((n->Opcode() == Op_ConvI2L && n->bottom_type() != TypeLong::LONG) ||
54 (n->Opcode() == Op_ConvL2I && n->bottom_type() != TypeInt::INT)) {
55 // ConvI2L/ConvL2I may have type information on it which is unsafe to push up
56 // so disable this for now
57 return nullptr;
58 }
59
60 // Splitting range check CastIIs through a loop induction Phi can
61 // cause new Phis to be created that are left unrelated to the loop
62 // induction Phi and prevent optimizations (vectorization)
63 if (n->Opcode() == Op_CastII && region->is_CountedLoop() &&
64 n->in(1) == region->as_CountedLoop()->phi()) {
65 return nullptr;
66 }
67
68 // Inline types should not be split through Phis because they cannot be merged
69 // through Phi nodes but each value input needs to be merged individually.
70 if (n->is_InlineType()) {
71 return nullptr;
72 }
73
74 if (cannot_split_division(n, region)) {
75 return nullptr;
76 }
77
78 int wins = 0;
79 assert(!n->is_CFG(), "");
80 assert(region->is_Region(), "");
81
82 const Type* type = n->bottom_type();
83 const TypeOopPtr* t_oop = _igvn.type(n)->isa_oopptr();
84 Node* phi;
85 if (t_oop != nullptr && t_oop->is_known_instance_field()) {
86 int iid = t_oop->instance_id();
87 int index = C->get_alias_index(t_oop);
88 int offset = t_oop->offset();
89 phi = new PhiNode(region, type, nullptr, iid, index, offset);
90 } else {
91 phi = PhiNode::make_blank(region, n);
92 }
93 uint old_unique = C->unique();
750 // CMOVE'd derived pointer? It's a CMOVE'd derived base. Thus
751 // CMOVE'ing a derived pointer requires we also CMOVE the base. If we
752 // have a Phi for the base here that we convert to a CMOVE all is well
753 // and good. But if the base is dead, we'll not make a CMOVE. Later
754 // the allocator will have to produce a base by creating a CMOVE of the
755 // relevant bases. This puts the allocator in the business of
756 // manufacturing expensive instructions, generally a bad plan.
757 // Just Say No to Conditionally-Moved Derived Pointers.
758 if (tp && tp->offset() != 0)
759 return nullptr;
760 cost++;
761 break;
762 }
763 default:
764 return nullptr; // In particular, can't do memory or I/O
765 }
766 // Add in cost any speculative ops
767 for (uint j = 1; j < region->req(); j++) {
768 Node *proj = region->in(j);
769 Node *inp = phi->in(j);
770 if (inp->isa_InlineType()) {
771 // TODO 8302217 This prevents PhiNode::push_inline_types_through
772 return nullptr;
773 }
774 if (get_ctrl(inp) == proj) { // Found local op
775 cost++;
776 // Check for a chain of dependent ops; these will all become
777 // speculative in a CMOV.
778 for (uint k = 1; k < inp->req(); k++)
779 if (get_ctrl(inp->in(k)) == proj)
780 cost += ConditionalMoveLimit; // Too much speculative goo
781 }
782 }
783 // See if the Phi is used by a Cmp or Narrow oop Decode/Encode.
784 // This will likely Split-If, a higher-payoff operation.
785 for (DUIterator_Fast kmax, k = phi->fast_outs(kmax); k < kmax; k++) {
786 Node* use = phi->fast_out(k);
787 if (use->is_Cmp() || use->is_DecodeNarrowPtr() || use->is_EncodeNarrowPtr())
788 cost += ConditionalMoveLimit;
789 // Is there a use inside the loop?
790 // Note: check only basic types since CMoveP is pinned.
791 if (!used_inside_loop && is_java_primitive(bt)) {
792 IdealLoopTree* u_loop = get_loop(has_ctrl(use) ? get_ctrl(use) : use);
793 if (r_loop == u_loop || r_loop->is_member(u_loop)) {
1082 assert(get_loop(lca)->_nest < n_loop->_nest || get_loop(lca)->_head->as_Loop()->is_in_infinite_subgraph(), "must not be moved into inner loop");
1083
1084 // Move store out of the loop
1085 _igvn.replace_node(hook, n->in(MemNode::Memory));
1086 _igvn.replace_input_of(n, 0, lca);
1087 set_ctrl_and_loop(n, lca);
1088
1089 // Disconnect the phi now. An empty phi can confuse other
1090 // optimizations in this pass of loop opts..
1091 if (phi->in(LoopNode::LoopBackControl) == phi) {
1092 _igvn.replace_node(phi, phi->in(LoopNode::EntryControl));
1093 n_loop->_body.yank(phi);
1094 }
1095 }
1096 }
1097 }
1098 }
1099 }
1100 }
1101
1102 // We can't use immutable memory for the flat array check because we are loading the mark word which is
1103 // mutable. Although the bits we are interested in are immutable (we check for markWord::unlocked_value),
1104 // we need to use raw memory to not break anti dependency analysis. Below code will attempt to still move
1105 // flat array checks out of loops, mainly to enable loop unswitching.
1106 void PhaseIdealLoop::move_flat_array_check_out_of_loop(Node* n) {
1107 // Skip checks for more than one array
1108 if (n->req() > 3) {
1109 return;
1110 }
1111 Node* mem = n->in(FlatArrayCheckNode::Memory);
1112 Node* array = n->in(FlatArrayCheckNode::ArrayOrKlass)->uncast();
1113 IdealLoopTree* check_loop = get_loop(get_ctrl(n));
1114 IdealLoopTree* ary_loop = get_loop(get_ctrl(array));
1115
1116 // Check if array is loop invariant
1117 if (!check_loop->is_member(ary_loop)) {
1118 // Walk up memory graph from the check until we leave the loop
1119 VectorSet wq;
1120 wq.set(mem->_idx);
1121 while (check_loop->is_member(get_loop(ctrl_or_self(mem)))) {
1122 if (mem->is_Phi()) {
1123 mem = mem->in(1);
1124 } else if (mem->is_MergeMem()) {
1125 mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
1126 } else if (mem->is_Proj()) {
1127 mem = mem->in(0);
1128 } else if (mem->is_MemBar() || mem->is_SafePoint()) {
1129 mem = mem->in(TypeFunc::Memory);
1130 } else if (mem->is_Store() || mem->is_LoadStore() || mem->is_ClearArray()) {
1131 mem = mem->in(MemNode::Memory);
1132 } else {
1133 #ifdef ASSERT
1134 mem->dump();
1135 #endif
1136 ShouldNotReachHere();
1137 }
1138 if (wq.test_set(mem->_idx)) {
1139 return;
1140 }
1141 }
1142 // Replace memory input and re-compute ctrl to move the check out of the loop
1143 _igvn.replace_input_of(n, 1, mem);
1144 set_ctrl_and_loop(n, get_early_ctrl(n));
1145 Node* bol = n->unique_out();
1146 set_ctrl_and_loop(bol, get_early_ctrl(bol));
1147 }
1148 }
1149
1150 // Split some nodes that take a counted loop phi as input at a counted
1151 // loop can cause vectorization of some expressions to fail
1152 bool PhaseIdealLoop::split_thru_phi_could_prevent_vectorization(Node* n, Node* n_blk) {
1153 if (!n_blk->is_CountedLoop()) {
1154 return false;
1155 }
1156
1157 int opcode = n->Opcode();
1158
1159 if (opcode != Op_AndI &&
1160 opcode != Op_MulI &&
1161 opcode != Op_RotateRight &&
1162 opcode != Op_RShiftI) {
1163 return false;
1164 }
1165
1166 return n->in(1) == n_blk->as_BaseCountedLoop()->phi();
1167 }
1168
1169 //------------------------------split_if_with_blocks_pre-----------------------
1170 // Do the real work in a non-recursive function. Data nodes want to be
1171 // cloned in the pre-order so they can feed each other nicely.
1172 Node *PhaseIdealLoop::split_if_with_blocks_pre( Node *n ) {
1173 // Cloning these guys is unlikely to win
1174 int n_op = n->Opcode();
1175 if (n_op == Op_MergeMem) {
1176 return n;
1177 }
1178 if (n->is_Proj()) {
1179 return n;
1180 }
1181
1182 if (n->isa_FlatArrayCheck()) {
1183 move_flat_array_check_out_of_loop(n);
1184 return n;
1185 }
1186
1187 // Do not clone-up CmpFXXX variations, as these are always
1188 // followed by a CmpI
1189 if (n->is_Cmp()) {
1190 return n;
1191 }
1192 // Attempt to use a conditional move instead of a phi/branch
1193 if (ConditionalMoveLimit > 0 && n_op == Op_Region) {
1194 Node *cmov = conditional_move( n );
1195 if (cmov) {
1196 return cmov;
1197 }
1198 }
1199 if (n->is_CFG() || n->is_LoadStore()) {
1200 return n;
1201 }
1202 if (n->is_Opaque1()) { // Opaque nodes cannot be mod'd
1203 if (!C->major_progress()) { // If chance of no more loop opts...
1204 _igvn._worklist.push(n); // maybe we'll remove them
1205 }
1206 return n;
1446
1447 return true;
1448 }
1449
1450 // Detect if the node is the inner strip-mined loop
1451 // Return: null if it's not the case, or the exit of outer strip-mined loop
1452 static Node* is_inner_of_stripmined_loop(const Node* out) {
1453 Node* out_le = nullptr;
1454
1455 if (out->is_CountedLoopEnd()) {
1456 const CountedLoopNode* loop = out->as_CountedLoopEnd()->loopnode();
1457
1458 if (loop != nullptr && loop->is_strip_mined()) {
1459 out_le = loop->in(LoopNode::EntryControl)->as_OuterStripMinedLoop()->outer_loop_exit();
1460 }
1461 }
1462
1463 return out_le;
1464 }
1465
1466 bool PhaseIdealLoop::flat_array_element_type_check(Node *n) {
1467 // If the CmpP is a subtype check for a value that has just been
1468 // loaded from an array, the subtype check guarantees the value
1469 // can't be stored in a flat array and the load of the value
1470 // happens with a flat array check then: push the type check
1471 // through the phi of the flat array check. This needs special
1472 // logic because the subtype check's input is not a phi but a
1473 // LoadKlass that must first be cloned through the phi.
1474 if (n->Opcode() != Op_CmpP) {
1475 return false;
1476 }
1477
1478 Node* klassptr = n->in(1);
1479 Node* klasscon = n->in(2);
1480
1481 if (klassptr->is_DecodeNarrowPtr()) {
1482 klassptr = klassptr->in(1);
1483 }
1484
1485 if (klassptr->Opcode() != Op_LoadKlass && klassptr->Opcode() != Op_LoadNKlass) {
1486 return false;
1487 }
1488
1489 if (!klasscon->is_Con()) {
1490 return false;
1491 }
1492
1493 Node* addr = klassptr->in(MemNode::Address);
1494
1495 if (!addr->is_AddP()) {
1496 return false;
1497 }
1498
1499 intptr_t offset;
1500 Node* obj = AddPNode::Ideal_base_and_offset(addr, &_igvn, offset);
1501
1502 if (obj == nullptr) {
1503 return false;
1504 }
1505
1506 assert(obj != nullptr && addr->in(AddPNode::Base) == addr->in(AddPNode::Address), "malformed AddP?");
1507 if (obj->Opcode() == Op_CastPP) {
1508 obj = obj->in(1);
1509 }
1510
1511 if (!obj->is_Phi()) {
1512 return false;
1513 }
1514
1515 Node* region = obj->in(0);
1516
1517 Node* phi = PhiNode::make_blank(region, n->in(1));
1518 for (uint i = 1; i < region->req(); i++) {
1519 Node* in = obj->in(i);
1520 Node* ctrl = region->in(i);
1521 if (addr->in(AddPNode::Base) != obj) {
1522 Node* cast = addr->in(AddPNode::Base);
1523 assert(cast->Opcode() == Op_CastPP && cast->in(0) != nullptr, "inconsistent subgraph");
1524 Node* cast_clone = cast->clone();
1525 cast_clone->set_req(0, ctrl);
1526 cast_clone->set_req(1, in);
1527 register_new_node(cast_clone, ctrl);
1528 const Type* tcast = cast_clone->Value(&_igvn);
1529 _igvn.set_type(cast_clone, tcast);
1530 cast_clone->as_Type()->set_type(tcast);
1531 in = cast_clone;
1532 }
1533 Node* addr_clone = addr->clone();
1534 addr_clone->set_req(AddPNode::Base, in);
1535 addr_clone->set_req(AddPNode::Address, in);
1536 register_new_node(addr_clone, ctrl);
1537 _igvn.set_type(addr_clone, addr_clone->Value(&_igvn));
1538 Node* klassptr_clone = klassptr->clone();
1539 klassptr_clone->set_req(2, addr_clone);
1540 register_new_node(klassptr_clone, ctrl);
1541 _igvn.set_type(klassptr_clone, klassptr_clone->Value(&_igvn));
1542 if (klassptr != n->in(1)) {
1543 Node* decode = n->in(1);
1544 assert(decode->is_DecodeNarrowPtr(), "inconsistent subgraph");
1545 Node* decode_clone = decode->clone();
1546 decode_clone->set_req(1, klassptr_clone);
1547 register_new_node(decode_clone, ctrl);
1548 _igvn.set_type(decode_clone, decode_clone->Value(&_igvn));
1549 klassptr_clone = decode_clone;
1550 }
1551 phi->set_req(i, klassptr_clone);
1552 }
1553 register_new_node(phi, region);
1554 Node* orig = n->in(1);
1555 _igvn.replace_input_of(n, 1, phi);
1556 split_if_with_blocks_post(n);
1557 if (n->outcnt() != 0) {
1558 _igvn.replace_input_of(n, 1, orig);
1559 _igvn.remove_dead_node(phi);
1560 }
1561 return true;
1562 }
1563
1564 //------------------------------split_if_with_blocks_post----------------------
1565 // Do the real work in a non-recursive function. CFG hackery wants to be
1566 // in the post-order, so it can dirty the I-DOM info and not use the dirtied
1567 // info.
1568 void PhaseIdealLoop::split_if_with_blocks_post(Node *n) {
1569
1570 if (flat_array_element_type_check(n)) {
1571 return;
1572 }
1573
1574 // Cloning Cmp through Phi's involves the split-if transform.
1575 // FastLock is not used by an If
1576 if (n->is_Cmp() && !n->is_FastLock()) {
1577 Node *n_ctrl = get_ctrl(n);
1578 // Determine if the Node has inputs from some local Phi.
1579 // Returns the block to clone thru.
1580 Node *n_blk = has_local_phi_input(n);
1581 if (n_blk != n_ctrl) {
1582 return;
1583 }
1584
1585 if (!can_split_if(n_ctrl)) {
1586 return;
1587 }
1588
1589 if (n->outcnt() != 1) {
1590 return; // Multiple bool's from 1 compare?
1591 }
1592 Node *bol = n->unique_out();
1593 assert(bol->is_Bool(), "expect a bool here");
1701 // 1. Move from RangeCheck "a" to RangeCheck "b": don't need to pin. If we ever remove b, then we pin
1702 // all its array accesses at that point.
1703 // 2. We move from RangeCheck "a" to regular if "b": need to pin. If we ever remove b, then its array
1704 // accesses would start to float, since we don't pin at that point.
1705 // 3. If we move from regular if: don't pin. All array accesses are already assumed to be pinned.
1706 bool pin_array_access_nodes = n->Opcode() == Op_RangeCheck &&
1707 prevdom->in(0)->Opcode() != Op_RangeCheck;
1708 dominated_by(prevdom->as_IfProj(), n->as_If(), false, pin_array_access_nodes);
1709 DEBUG_ONLY( if (VerifyLoopOptimizations) { verify(); } );
1710 return;
1711 }
1712 prevdom = dom;
1713 dom = idom(prevdom);
1714 }
1715 }
1716 }
1717
1718 try_sink_out_of_loop(n);
1719
1720 try_move_store_after_loop(n);
1721
1722 // Remove multiple allocations of the same inline type
1723 if (n->is_InlineType()) {
1724 n->as_InlineType()->remove_redundant_allocations(this);
1725 }
1726 }
1727
1728 // Transform:
1729 //
1730 // if (some_condition) {
1731 // // body 1
1732 // } else {
1733 // // body 2
1734 // }
1735 // if (some_condition) {
1736 // // body 3
1737 // } else {
1738 // // body 4
1739 // }
1740 //
1741 // into:
1742 //
1743 //
1744 // if (some_condition) {
1745 // // body 1
2190 uint i;
2191 for (i = 1; i < phi->req(); i++) {
2192 Node* b = phi->in(i);
2193 if (b->is_Phi()) {
2194 _igvn.replace_input_of(phi, i, clone_iff(b->as_Phi()));
2195 } else {
2196 assert(b->is_Bool() || b->is_OpaqueNotNull() || b->is_OpaqueInitializedAssertionPredicate(),
2197 "bool, non-null check with OpaqueNotNull or Initialized Assertion Predicate with its Opaque node");
2198 }
2199 }
2200 Node* n = phi->in(1);
2201 Node* sample_opaque = nullptr;
2202 Node *sample_bool = nullptr;
2203 if (n->is_OpaqueNotNull() || n->is_OpaqueInitializedAssertionPredicate()) {
2204 sample_opaque = n;
2205 sample_bool = n->in(1);
2206 assert(sample_bool->is_Bool(), "wrong type");
2207 } else {
2208 sample_bool = n;
2209 }
2210 Node* sample_cmp = sample_bool->in(1);
2211 const Type* t = Type::TOP;
2212 const TypePtr* at = nullptr;
2213 if (sample_cmp->is_FlatArrayCheck()) {
2214 // Left input of a FlatArrayCheckNode is memory, set the (adr) type of the phi accordingly
2215 assert(sample_cmp->in(1)->bottom_type() == Type::MEMORY, "unexpected input type");
2216 t = Type::MEMORY;
2217 at = TypeRawPtr::BOTTOM;
2218 }
2219
2220 // Make Phis to merge the Cmp's inputs.
2221 PhiNode *phi1 = new PhiNode(phi->in(0), t, at);
2222 PhiNode *phi2 = new PhiNode(phi->in(0), Type::TOP);
2223 for (i = 1; i < phi->req(); i++) {
2224 Node *n1 = sample_opaque == nullptr ? phi->in(i)->in(1)->in(1) : phi->in(i)->in(1)->in(1)->in(1);
2225 Node *n2 = sample_opaque == nullptr ? phi->in(i)->in(1)->in(2) : phi->in(i)->in(1)->in(1)->in(2);
2226 phi1->set_req(i, n1);
2227 phi2->set_req(i, n2);
2228 phi1->set_type(phi1->type()->meet_speculative(n1->bottom_type()));
2229 phi2->set_type(phi2->type()->meet_speculative(n2->bottom_type()));
2230 }
2231 // See if these Phis have been made before.
2232 // Register with optimizer
2233 Node *hit1 = _igvn.hash_find_insert(phi1);
2234 if (hit1) { // Hit, toss just made Phi
2235 _igvn.remove_dead_node(phi1); // Remove new phi
2236 assert(hit1->is_Phi(), "" );
2237 phi1 = (PhiNode*)hit1; // Use existing phi
2238 } else { // Miss
2239 _igvn.register_new_node_with_optimizer(phi1);
2240 }
2241 Node *hit2 = _igvn.hash_find_insert(phi2);
|