< prev index next >

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

Print this page

 436           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 437         "sha512_implCompressMB",
 438         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
 439           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 440         "encodeBlock",
 441         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 442           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 443         "decodeBlock",
 444         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 445           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 446         "intpoly_montgomeryMult_P256",
 447         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahLoad  },   { TypeFunc::Parms+2, ShenandoahStore },
 448           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 449         "intpoly_assign",
 450         { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },  { -1, ShenandoahNone },
 451           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 452       };
 453 
 454       if (call->is_call_to_arraycopystub()) {
 455         Node* dest = nullptr;
 456         const TypeTuple* args = n->as_Call()->_tf->domain();
 457         for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
 458           if (args->field_at(i)->isa_ptr()) {
 459             j++;
 460             if (j == 2) {
 461               dest = n->in(i);
 462               break;
 463             }
 464           }
 465         }
 466         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
 467             !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
 468           report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
 469         }
 470       } else if (strlen(call->_name) > 5 &&
 471                  !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
 472         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
 473           report_verify_failure("Shenandoah verification: _fill should have barriers", n);
 474         }
 475       } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
 476         // skip

 557         { { 2, ShenandoahLoad },                  { 3, ShenandoahLoad } },
 558         Op_VectorizedHashCode,
 559         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
 560         Op_EncodeISOArray,
 561         { { 2, ShenandoahLoad },                  { 3, ShenandoahStore } },
 562         Op_CountPositives,
 563         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone} },
 564         Op_CastP2X,
 565         { { 1, ShenandoahLoad },                  { -1, ShenandoahNone} },
 566         Op_StrIndexOfChar,
 567         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
 568       };
 569 
 570       const int others_len = sizeof(others) / sizeof(others[0]);
 571       int i = 0;
 572       for (; i < others_len; i++) {
 573         if (others[i].opcode == n->Opcode()) {
 574           break;
 575         }
 576       }
 577       uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
 578       if (i != others_len) {
 579         const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
 580         for (uint j = 0; j < inputs_len; j++) {
 581           int pos = others[i].inputs[j].pos;
 582           if (pos == -1) {
 583             break;
 584           }
 585           if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
 586             report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
 587           }
 588         }
 589         for (uint j = 1; j < stop; j++) {
 590           if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
 591               n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
 592             uint k = 0;
 593             for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
 594             if (k == inputs_len) {
 595               fatal("arg %d for node %s not covered", j, n->Name());
 596             }
 597           }

 777     mem_ctrl = phase->ctrl_or_self(mem);
 778   }
 779   return mem;
 780 }
 781 
 782 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
 783   Node* mem = nullptr;
 784   Node* c = ctrl;
 785   do {
 786     if (c->is_Region()) {
 787       for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
 788         Node* u = c->fast_out(i);
 789         if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
 790           if (u->adr_type() == TypePtr::BOTTOM) {
 791             mem = u;
 792           }
 793         }
 794       }
 795     } else {
 796       if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
 797         CallProjections projs;
 798         c->as_Call()->extract_projections(&projs, true, false);
 799         if (projs.fallthrough_memproj != nullptr) {
 800           if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
 801             if (projs.catchall_memproj == nullptr) {
 802               mem = projs.fallthrough_memproj;
 803             } else {
 804               if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
 805                 mem = projs.fallthrough_memproj;
 806               } else {
 807                 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
 808                 mem = projs.catchall_memproj;
 809               }
 810             }
 811           }
 812         } else {
 813           Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
 814           if (proj != nullptr &&
 815               proj->adr_type() == TypePtr::BOTTOM) {
 816             mem = proj;
 817           }
 818         }
 819       } else {
 820         for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
 821           Node* u = c->fast_out(i);
 822           if (u->is_Proj() &&
 823               u->bottom_type() == Type::MEMORY &&
 824               u->adr_type() == TypePtr::BOTTOM) {
 825               assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
 826               assert(mem == nullptr, "only one proj");
 827               mem = u;
 828           }

1049       if (c != ctrl ||
1050           is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1051           ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1052         phase->igvn().rehash_node_delayed(u);
1053         int nb = u->replace_edge(ctrl, region, &phase->igvn());
1054         if (u->is_CFG()) {
1055           if (phase->idom(u) == ctrl) {
1056             phase->set_idom(u, region, phase->dom_depth(region));
1057           }
1058         } else if (phase->get_ctrl(u) == ctrl) {
1059           assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1060           uses.push(u);
1061         }
1062         assert(nb == 1, "more than 1 ctrl input?");
1063         --i, imax -= nb;
1064       }
1065     }
1066   }
1067 }
1068 
1069 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
1070   Node* region = nullptr;
1071   while (c != ctrl) {
1072     if (c->is_Region()) {
1073       region = c;
1074     }
1075     c = phase->idom(c);
1076   }
1077   assert(region != nullptr, "");
1078   Node* phi = new PhiNode(region, n->bottom_type());
1079   for (uint j = 1; j < region->req(); j++) {
1080     Node* in = region->in(j);
1081     if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1082       phi->init_req(j, n);
1083     } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1084       phi->init_req(j, n_clone);
1085     } else {
1086       phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1087     }
1088   }
1089   phase->register_new_node(phi, region);
1090   return phi;
1091 }
1092 
1093 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1094   ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1095 
1096   Unique_Node_List uses;
1097   Node_Stack stack(0);
1098   Node_List clones;
1099   for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1100     ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1101 
1102     Node* ctrl = phase->get_ctrl(lrb);
1103     Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);

1166               if (phase->has_ctrl(in)) {
1167                 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1168 #ifdef ASSERT
1169                   for (uint i = 0; i < stack.size(); i++) {
1170                     assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1171                   }
1172 #endif
1173                   stack.push(in, 0);
1174                 }
1175               } else {
1176                 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1177               }
1178             }
1179           } else {
1180             phase->set_ctrl(n, call->in(0));
1181             stack.pop();
1182           }
1183         } while(stack.size() > 0);
1184         continue;
1185       }
1186       CallProjections projs;
1187       call->extract_projections(&projs, false, false);
1188 
1189       // If this is a runtime call, it doesn't have an exception handling path
1190       if (projs.fallthrough_catchproj == nullptr) {
1191         assert(call->method() == nullptr, "should be runtime call");
1192         assert(projs.catchall_catchproj == nullptr, "runtime call should not have catch all projection");
1193         continue;
1194       }
1195 
1196       // Otherwise, clone the barrier so there's one for the fallthrough and one for the exception handling path
1197 #ifdef ASSERT
1198       VectorSet cloned;
1199 #endif
1200       Node* lrb_clone = lrb->clone();
1201       phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1202       phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1203 
1204       stack.push(lrb, 0);
1205       clones.push(lrb_clone);
1206 
1207       do {
1208         assert(stack.size() == clones.size(), "");
1209         Node* n = stack.node();
1210 #ifdef ASSERT
1211         if (n->is_Load()) {
1212           Node* mem = n->in(MemNode::Memory);
1213           for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1214             Node* u = mem->fast_out(j);
1215             assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1216           }
1217         }
1218 #endif
1219         uint idx = stack.index();
1220         Node* n_clone = clones.at(clones.size()-1);
1221         if (idx < n->outcnt()) {
1222           Node* u = n->raw_out(idx);
1223           Node* c = phase->ctrl_or_self(u);
1224           if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) {
1225             stack.set_index(idx+1);
1226             assert(!u->is_CFG(), "");
1227             stack.push(u, 0);
1228             assert(!cloned.test_set(u->_idx), "only one clone");
1229             Node* u_clone = u->clone();
1230             int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1231             assert(nb > 0, "should have replaced some uses");
1232             phase->register_new_node(u_clone, projs.catchall_catchproj);
1233             clones.push(u_clone);
1234             phase->set_ctrl(u, projs.fallthrough_catchproj);
1235           } else {
1236             bool replaced = false;
1237             if (u->is_Phi()) {
1238               for (uint k = 1; k < u->req(); k++) {
1239                 if (u->in(k) == n) {
1240                   if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1241                     phase->igvn().replace_input_of(u, k, n_clone);
1242                     replaced = true;
1243                   } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1244                     phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1245                     replaced = true;
1246                   }
1247                 }
1248               }
1249             } else {
1250               if (phase->is_dominator(projs.catchall_catchproj, c)) {
1251                 phase->igvn().rehash_node_delayed(u);
1252                 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1253                 assert(nb > 0, "should have replaced some uses");
1254                 replaced = true;
1255               } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1256                 if (u->is_If()) {
1257                   // Can't break If/Bool/Cmp chain
1258                   assert(n->is_Bool(), "unexpected If shape");
1259                   assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1260                   assert(n_clone->is_Bool(), "unexpected clone");
1261                   assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1262                   Node* bol_clone = n->clone();
1263                   Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1264                   bol_clone->set_req(1, cmp_clone);
1265 
1266                   Node* nn = stack.node_at(stack.size()-3);
1267                   Node* nn_clone = clones.at(clones.size()-3);
1268                   assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1269 
1270                   int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1271                                                    &phase->igvn());
1272                   assert(nb > 0, "should have replaced some uses");
1273 
1274                   phase->register_new_node(bol_clone, u->in(0));
1275                   phase->register_new_node(cmp_clone, u->in(0));

2056             } else {
2057               assert(c != c->in(0), "");
2058               c = c->in(0);
2059             }
2060           }
2061         }
2062       }
2063     } while (stack.size() > 0);
2064     assert(mem != nullptr, "should have found safepoint");
2065   } else {
2066     mem = phi_mem;
2067   }
2068   return mem;
2069 }
2070 
2071 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
2072   Node* c = _phase->get_ctrl(n);
2073   if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
2074     assert(c == n->in(0), "");
2075     CallNode* call = c->as_Call();
2076     CallProjections projs;
2077     call->extract_projections(&projs, true, false);
2078     if (projs.catchall_memproj != nullptr) {
2079       if (projs.fallthrough_memproj == n) {
2080         c = projs.fallthrough_catchproj;
2081       } else {
2082         assert(projs.catchall_memproj == n, "");
2083         c = projs.catchall_catchproj;
2084       }
2085     }
2086   }
2087   return c;
2088 }
2089 
2090 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
2091   if (_phase->has_ctrl(n))
2092     return get_ctrl(n);
2093   else {
2094     assert (n->is_CFG(), "must be a CFG node");
2095     return n;
2096   }
2097 }
2098 
2099 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2100   return m != nullptr && get_ctrl(m) == c;
2101 }
2102 
2103 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {

 436           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 437         "sha512_implCompressMB",
 438         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
 439           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 440         "encodeBlock",
 441         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 442           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 443         "decodeBlock",
 444         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 445           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 446         "intpoly_montgomeryMult_P256",
 447         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahLoad  },   { TypeFunc::Parms+2, ShenandoahStore },
 448           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 449         "intpoly_assign",
 450         { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },  { -1, ShenandoahNone },
 451           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 452       };
 453 
 454       if (call->is_call_to_arraycopystub()) {
 455         Node* dest = nullptr;
 456         const TypeTuple* args = n->as_Call()->_tf->domain_sig();
 457         for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
 458           if (args->field_at(i)->isa_ptr()) {
 459             j++;
 460             if (j == 2) {
 461               dest = n->in(i);
 462               break;
 463             }
 464           }
 465         }
 466         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
 467             !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
 468           report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
 469         }
 470       } else if (strlen(call->_name) > 5 &&
 471                  !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
 472         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
 473           report_verify_failure("Shenandoah verification: _fill should have barriers", n);
 474         }
 475       } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
 476         // skip

 557         { { 2, ShenandoahLoad },                  { 3, ShenandoahLoad } },
 558         Op_VectorizedHashCode,
 559         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
 560         Op_EncodeISOArray,
 561         { { 2, ShenandoahLoad },                  { 3, ShenandoahStore } },
 562         Op_CountPositives,
 563         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone} },
 564         Op_CastP2X,
 565         { { 1, ShenandoahLoad },                  { -1, ShenandoahNone} },
 566         Op_StrIndexOfChar,
 567         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
 568       };
 569 
 570       const int others_len = sizeof(others) / sizeof(others[0]);
 571       int i = 0;
 572       for (; i < others_len; i++) {
 573         if (others[i].opcode == n->Opcode()) {
 574           break;
 575         }
 576       }
 577       uint stop = n->is_Call() ? n->as_Call()->tf()->domain_sig()->cnt() : n->req();
 578       if (i != others_len) {
 579         const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
 580         for (uint j = 0; j < inputs_len; j++) {
 581           int pos = others[i].inputs[j].pos;
 582           if (pos == -1) {
 583             break;
 584           }
 585           if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
 586             report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
 587           }
 588         }
 589         for (uint j = 1; j < stop; j++) {
 590           if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
 591               n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
 592             uint k = 0;
 593             for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
 594             if (k == inputs_len) {
 595               fatal("arg %d for node %s not covered", j, n->Name());
 596             }
 597           }

 777     mem_ctrl = phase->ctrl_or_self(mem);
 778   }
 779   return mem;
 780 }
 781 
 782 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
 783   Node* mem = nullptr;
 784   Node* c = ctrl;
 785   do {
 786     if (c->is_Region()) {
 787       for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
 788         Node* u = c->fast_out(i);
 789         if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
 790           if (u->adr_type() == TypePtr::BOTTOM) {
 791             mem = u;
 792           }
 793         }
 794       }
 795     } else {
 796       if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
 797         CallProjections* projs = c->as_Call()->extract_projections(true, false);
 798         if (projs->fallthrough_memproj != nullptr) {
 799           if (projs->fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
 800             if (projs->catchall_memproj == nullptr) {
 801               mem = projs->fallthrough_memproj;

 802             } else {
 803               if (phase->is_dominator(projs->fallthrough_catchproj, ctrl)) {
 804                 mem = projs->fallthrough_memproj;
 805               } else {
 806                 assert(phase->is_dominator(projs->catchall_catchproj, ctrl), "one proj must dominate barrier");
 807                 mem = projs->catchall_memproj;
 808               }
 809             }
 810           }
 811         } else {
 812           Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
 813           if (proj != nullptr &&
 814               proj->adr_type() == TypePtr::BOTTOM) {
 815             mem = proj;
 816           }
 817         }
 818       } else {
 819         for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
 820           Node* u = c->fast_out(i);
 821           if (u->is_Proj() &&
 822               u->bottom_type() == Type::MEMORY &&
 823               u->adr_type() == TypePtr::BOTTOM) {
 824               assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
 825               assert(mem == nullptr, "only one proj");
 826               mem = u;
 827           }

1048       if (c != ctrl ||
1049           is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1050           ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1051         phase->igvn().rehash_node_delayed(u);
1052         int nb = u->replace_edge(ctrl, region, &phase->igvn());
1053         if (u->is_CFG()) {
1054           if (phase->idom(u) == ctrl) {
1055             phase->set_idom(u, region, phase->dom_depth(region));
1056           }
1057         } else if (phase->get_ctrl(u) == ctrl) {
1058           assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1059           uses.push(u);
1060         }
1061         assert(nb == 1, "more than 1 ctrl input?");
1062         --i, imax -= nb;
1063       }
1064     }
1065   }
1066 }
1067 
1068 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections* projs, PhaseIdealLoop* phase) {
1069   Node* region = nullptr;
1070   while (c != ctrl) {
1071     if (c->is_Region()) {
1072       region = c;
1073     }
1074     c = phase->idom(c);
1075   }
1076   assert(region != nullptr, "");
1077   Node* phi = new PhiNode(region, n->bottom_type());
1078   for (uint j = 1; j < region->req(); j++) {
1079     Node* in = region->in(j);
1080     if (phase->is_dominator(projs->fallthrough_catchproj, in)) {
1081       phi->init_req(j, n);
1082     } else if (phase->is_dominator(projs->catchall_catchproj, in)) {
1083       phi->init_req(j, n_clone);
1084     } else {
1085       phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1086     }
1087   }
1088   phase->register_new_node(phi, region);
1089   return phi;
1090 }
1091 
1092 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1093   ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1094 
1095   Unique_Node_List uses;
1096   Node_Stack stack(0);
1097   Node_List clones;
1098   for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1099     ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1100 
1101     Node* ctrl = phase->get_ctrl(lrb);
1102     Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);

1165               if (phase->has_ctrl(in)) {
1166                 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1167 #ifdef ASSERT
1168                   for (uint i = 0; i < stack.size(); i++) {
1169                     assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1170                   }
1171 #endif
1172                   stack.push(in, 0);
1173                 }
1174               } else {
1175                 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1176               }
1177             }
1178           } else {
1179             phase->set_ctrl(n, call->in(0));
1180             stack.pop();
1181           }
1182         } while(stack.size() > 0);
1183         continue;
1184       }
1185       CallProjections* projs = call->extract_projections(false, false);

1186 
1187       // If this is a runtime call, it doesn't have an exception handling path
1188       if (projs->fallthrough_catchproj == nullptr) {
1189         assert(call->method() == nullptr, "should be runtime call");
1190         assert(projs->catchall_catchproj == nullptr, "runtime call should not have catch all projection");
1191         continue;
1192       }
1193 
1194       // Otherwise, clone the barrier so there's one for the fallthrough and one for the exception handling path
1195 #ifdef ASSERT
1196       VectorSet cloned;
1197 #endif
1198       Node* lrb_clone = lrb->clone();
1199       phase->register_new_node(lrb_clone, projs->catchall_catchproj);
1200       phase->set_ctrl(lrb, projs->fallthrough_catchproj);
1201 
1202       stack.push(lrb, 0);
1203       clones.push(lrb_clone);
1204 
1205       do {
1206         assert(stack.size() == clones.size(), "");
1207         Node* n = stack.node();
1208 #ifdef ASSERT
1209         if (n->is_Load()) {
1210           Node* mem = n->in(MemNode::Memory);
1211           for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1212             Node* u = mem->fast_out(j);
1213             assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1214           }
1215         }
1216 #endif
1217         uint idx = stack.index();
1218         Node* n_clone = clones.at(clones.size()-1);
1219         if (idx < n->outcnt()) {
1220           Node* u = n->raw_out(idx);
1221           Node* c = phase->ctrl_or_self(u);
1222           if (phase->is_dominator(call, c) && phase->is_dominator(c, projs->fallthrough_proj)) {
1223             stack.set_index(idx+1);
1224             assert(!u->is_CFG(), "");
1225             stack.push(u, 0);
1226             assert(!cloned.test_set(u->_idx), "only one clone");
1227             Node* u_clone = u->clone();
1228             int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1229             assert(nb > 0, "should have replaced some uses");
1230             phase->register_new_node(u_clone, projs->catchall_catchproj);
1231             clones.push(u_clone);
1232             phase->set_ctrl(u, projs->fallthrough_catchproj);
1233           } else {
1234             bool replaced = false;
1235             if (u->is_Phi()) {
1236               for (uint k = 1; k < u->req(); k++) {
1237                 if (u->in(k) == n) {
1238                   if (phase->is_dominator(projs->catchall_catchproj, u->in(0)->in(k))) {
1239                     phase->igvn().replace_input_of(u, k, n_clone);
1240                     replaced = true;
1241                   } else if (!phase->is_dominator(projs->fallthrough_catchproj, u->in(0)->in(k))) {
1242                     phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1243                     replaced = true;
1244                   }
1245                 }
1246               }
1247             } else {
1248               if (phase->is_dominator(projs->catchall_catchproj, c)) {
1249                 phase->igvn().rehash_node_delayed(u);
1250                 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1251                 assert(nb > 0, "should have replaced some uses");
1252                 replaced = true;
1253               } else if (!phase->is_dominator(projs->fallthrough_catchproj, c)) {
1254                 if (u->is_If()) {
1255                   // Can't break If/Bool/Cmp chain
1256                   assert(n->is_Bool(), "unexpected If shape");
1257                   assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1258                   assert(n_clone->is_Bool(), "unexpected clone");
1259                   assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1260                   Node* bol_clone = n->clone();
1261                   Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1262                   bol_clone->set_req(1, cmp_clone);
1263 
1264                   Node* nn = stack.node_at(stack.size()-3);
1265                   Node* nn_clone = clones.at(clones.size()-3);
1266                   assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1267 
1268                   int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1269                                                    &phase->igvn());
1270                   assert(nb > 0, "should have replaced some uses");
1271 
1272                   phase->register_new_node(bol_clone, u->in(0));
1273                   phase->register_new_node(cmp_clone, u->in(0));

2054             } else {
2055               assert(c != c->in(0), "");
2056               c = c->in(0);
2057             }
2058           }
2059         }
2060       }
2061     } while (stack.size() > 0);
2062     assert(mem != nullptr, "should have found safepoint");
2063   } else {
2064     mem = phi_mem;
2065   }
2066   return mem;
2067 }
2068 
2069 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
2070   Node* c = _phase->get_ctrl(n);
2071   if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
2072     assert(c == n->in(0), "");
2073     CallNode* call = c->as_Call();
2074     CallProjections* projs = call->extract_projections(true, false);
2075     if (projs->catchall_memproj != nullptr) {
2076       if (projs->fallthrough_memproj == n) {
2077         c = projs->fallthrough_catchproj;

2078       } else {
2079         assert(projs->catchall_memproj == n, "");
2080         c = projs->catchall_catchproj;
2081       }
2082     }
2083   }
2084   return c;
2085 }
2086 
2087 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
2088   if (_phase->has_ctrl(n))
2089     return get_ctrl(n);
2090   else {
2091     assert (n->is_CFG(), "must be a CFG node");
2092     return n;
2093   }
2094 }
2095 
2096 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2097   return m != nullptr && get_ctrl(m) == c;
2098 }
2099 
2100 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
< prev index next >