< prev index next >

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

Print this page

 430           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 431         "sha512_implCompressMB",
 432         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
 433           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 434         "encodeBlock",
 435         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 436           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 437         "decodeBlock",
 438         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 439           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 440         "intpoly_montgomeryMult_P256",
 441         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahLoad  },   { TypeFunc::Parms+2, ShenandoahStore },
 442           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 443         "intpoly_assign",
 444         { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },  { -1, ShenandoahNone },
 445           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 446       };
 447 
 448       if (call->is_call_to_arraycopystub()) {
 449         Node* dest = nullptr;
 450         const TypeTuple* args = n->as_Call()->_tf->domain();
 451         for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
 452           if (args->field_at(i)->isa_ptr()) {
 453             j++;
 454             if (j == 2) {
 455               dest = n->in(i);
 456               break;
 457             }
 458           }
 459         }
 460         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
 461             !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
 462           report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
 463         }
 464       } else if (strlen(call->_name) > 5 &&
 465                  !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
 466         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
 467           report_verify_failure("Shenandoah verification: _fill should have barriers", n);
 468         }
 469       } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
 470         // skip

 551         { { 2, ShenandoahLoad },                  { 3, ShenandoahLoad } },
 552         Op_VectorizedHashCode,
 553         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
 554         Op_EncodeISOArray,
 555         { { 2, ShenandoahLoad },                  { 3, ShenandoahStore } },
 556         Op_CountPositives,
 557         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone} },
 558         Op_CastP2X,
 559         { { 1, ShenandoahLoad },                  { -1, ShenandoahNone} },
 560         Op_StrIndexOfChar,
 561         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
 562       };
 563 
 564       const int others_len = sizeof(others) / sizeof(others[0]);
 565       int i = 0;
 566       for (; i < others_len; i++) {
 567         if (others[i].opcode == n->Opcode()) {
 568           break;
 569         }
 570       }
 571       uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
 572       if (i != others_len) {
 573         const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
 574         for (uint j = 0; j < inputs_len; j++) {
 575           int pos = others[i].inputs[j].pos;
 576           if (pos == -1) {
 577             break;
 578           }
 579           if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
 580             report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
 581           }
 582         }
 583         for (uint j = 1; j < stop; j++) {
 584           if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
 585               n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
 586             uint k = 0;
 587             for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
 588             if (k == inputs_len) {
 589               fatal("arg %d for node %s not covered", j, n->Name());
 590             }
 591           }

 786     mem_ctrl = phase->ctrl_or_self(mem);
 787   }
 788   return mem;
 789 }
 790 
 791 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
 792   Node* mem = nullptr;
 793   Node* c = ctrl;
 794   do {
 795     if (c->is_Region()) {
 796       for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
 797         Node* u = c->fast_out(i);
 798         if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
 799           if (u->adr_type() == TypePtr::BOTTOM) {
 800             mem = u;
 801           }
 802         }
 803       }
 804     } else {
 805       if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
 806         CallProjections projs;
 807         c->as_Call()->extract_projections(&projs, true, false);
 808         if (projs.fallthrough_memproj != nullptr) {
 809           if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
 810             if (projs.catchall_memproj == nullptr) {
 811               mem = projs.fallthrough_memproj;
 812             } else {
 813               if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
 814                 mem = projs.fallthrough_memproj;
 815               } else {
 816                 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
 817                 mem = projs.catchall_memproj;
 818               }
 819             }
 820           }
 821         } else {
 822           Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
 823           if (proj != nullptr &&
 824               proj->adr_type() == TypePtr::BOTTOM) {
 825             mem = proj;
 826           }
 827         }
 828       } else {
 829         for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
 830           Node* u = c->fast_out(i);
 831           if (u->is_Proj() &&
 832               u->bottom_type() == Type::MEMORY &&
 833               u->adr_type() == TypePtr::BOTTOM) {
 834               assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
 835               assert(mem == nullptr, "only one proj");
 836               mem = u;
 837           }

1058       if (old_c != ctrl ||
1059           is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1060           ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1061         phase->igvn().rehash_node_delayed(u);
1062         int nb = u->replace_edge(ctrl, region, &phase->igvn());
1063         if (u->is_CFG()) {
1064           if (phase->idom(u) == ctrl) {
1065             phase->set_idom(u, region, phase->dom_depth(region));
1066           }
1067         } else if (phase->get_ctrl(u) == ctrl) {
1068           assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1069           uses.push(u);
1070         }
1071         assert(nb == 1, "more than 1 ctrl input?");
1072         --i, imax -= nb;
1073       }
1074     }
1075   }
1076 }
1077 
1078 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
1079   Node* region = nullptr;
1080   while (c != ctrl) {
1081     if (c->is_Region()) {
1082       region = c;
1083     }
1084     c = phase->idom(c);
1085   }
1086   assert(region != nullptr, "");
1087   Node* phi = new PhiNode(region, n->bottom_type());
1088   for (uint j = 1; j < region->req(); j++) {
1089     Node* in = region->in(j);
1090     if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1091       phi->init_req(j, n);
1092     } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1093       phi->init_req(j, n_clone);
1094     } else {
1095       phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1096     }
1097   }
1098   phase->register_new_node(phi, region);
1099   return phi;
1100 }
1101 
1102 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1103   ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1104 
1105   Unique_Node_List uses;
1106   Node_Stack stack(0);
1107   Node_List clones;
1108   for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1109     ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1110 
1111     Node* ctrl = phase->get_ctrl(lrb);
1112     Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);

1175               if (phase->has_ctrl(in)) {
1176                 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1177 #ifdef ASSERT
1178                   for (uint i = 0; i < stack.size(); i++) {
1179                     assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1180                   }
1181 #endif
1182                   stack.push(in, 0);
1183                 }
1184               } else {
1185                 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1186               }
1187             }
1188           } else {
1189             phase->set_ctrl(n, call->in(0));
1190             stack.pop();
1191           }
1192         } while(stack.size() > 0);
1193         continue;
1194       }
1195       CallProjections projs;
1196       call->extract_projections(&projs, false, false);
1197 
1198       // If this is a runtime call, it doesn't have an exception handling path
1199       if (projs.fallthrough_catchproj == nullptr) {
1200         assert(call->method() == nullptr, "should be runtime call");
1201         assert(projs.catchall_catchproj == nullptr, "runtime call should not have catch all projection");
1202         continue;
1203       }
1204 
1205       // Otherwise, clone the barrier so there's one for the fallthrough and one for the exception handling path
1206 #ifdef ASSERT
1207       VectorSet cloned;
1208 #endif
1209       Node* lrb_clone = lrb->clone();
1210       phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1211       phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1212 
1213       stack.push(lrb, 0);
1214       clones.push(lrb_clone);
1215 
1216       do {
1217         assert(stack.size() == clones.size(), "");
1218         Node* n = stack.node();
1219 #ifdef ASSERT
1220         if (n->is_Load()) {
1221           Node* mem = n->in(MemNode::Memory);
1222           for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1223             Node* u = mem->fast_out(j);
1224             assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1225           }
1226         }
1227 #endif
1228         uint idx = stack.index();
1229         Node* n_clone = clones.at(clones.size()-1);
1230         if (idx < n->outcnt()) {
1231           Node* u = n->raw_out(idx);
1232           Node* c = phase->ctrl_or_self(u);
1233           if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) {
1234             stack.set_index(idx+1);
1235             assert(!u->is_CFG(), "");
1236             stack.push(u, 0);
1237             assert(!cloned.test_set(u->_idx), "only one clone");
1238             Node* u_clone = u->clone();
1239             int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1240             assert(nb > 0, "should have replaced some uses");
1241             phase->register_new_node(u_clone, projs.catchall_catchproj);
1242             clones.push(u_clone);
1243             phase->set_ctrl(u, projs.fallthrough_catchproj);
1244           } else {
1245             bool replaced = false;
1246             if (u->is_Phi()) {
1247               for (uint k = 1; k < u->req(); k++) {
1248                 if (u->in(k) == n) {
1249                   if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1250                     phase->igvn().replace_input_of(u, k, n_clone);
1251                     replaced = true;
1252                   } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1253                     phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1254                     replaced = true;
1255                   }
1256                 }
1257               }
1258             } else {
1259               if (phase->is_dominator(projs.catchall_catchproj, c)) {
1260                 phase->igvn().rehash_node_delayed(u);
1261                 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1262                 assert(nb > 0, "should have replaced some uses");
1263                 replaced = true;
1264               } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1265                 if (u->is_If()) {
1266                   // Can't break If/Bool/Cmp chain
1267                   assert(n->is_Bool(), "unexpected If shape");
1268                   assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1269                   assert(n_clone->is_Bool(), "unexpected clone");
1270                   assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1271                   Node* bol_clone = n->clone();
1272                   Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1273                   bol_clone->set_req(1, cmp_clone);
1274 
1275                   Node* nn = stack.node_at(stack.size()-3);
1276                   Node* nn_clone = clones.at(clones.size()-3);
1277                   assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1278 
1279                   int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1280                                                    &phase->igvn());
1281                   assert(nb > 0, "should have replaced some uses");
1282 
1283                   phase->register_new_node(bol_clone, u->in(0));
1284                   phase->register_new_node(cmp_clone, u->in(0));

1835             } else {
1836               assert(c != c->in(0), "");
1837               c = c->in(0);
1838             }
1839           }
1840         }
1841       }
1842     } while (stack.size() > 0);
1843     assert(mem != nullptr, "should have found safepoint");
1844   } else {
1845     mem = phi_mem;
1846   }
1847   return mem;
1848 }
1849 
1850 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
1851   Node* c = _phase->get_ctrl(n);
1852   if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
1853     assert(c == n->in(0), "");
1854     CallNode* call = c->as_Call();
1855     CallProjections projs;
1856     call->extract_projections(&projs, true, false);
1857     if (projs.catchall_memproj != nullptr) {
1858       if (projs.fallthrough_memproj == n) {
1859         c = projs.fallthrough_catchproj;
1860       } else {
1861         assert(projs.catchall_memproj == n, "");
1862         c = projs.catchall_catchproj;
1863       }
1864     }
1865   }
1866   return c;
1867 }
1868 
1869 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
1870   if (_phase->has_ctrl(n))
1871     return get_ctrl(n);
1872   else {
1873     assert (n->is_CFG(), "must be a CFG node");
1874     return n;
1875   }
1876 }
1877 
1878 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
1879   return m != nullptr && get_ctrl(m) == c;
1880 }
1881 
1882 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {

 430           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 431         "sha512_implCompressMB",
 432         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
 433           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 434         "encodeBlock",
 435         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 436           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 437         "decodeBlock",
 438         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 439           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 440         "intpoly_montgomeryMult_P256",
 441         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahLoad  },   { TypeFunc::Parms+2, ShenandoahStore },
 442           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 443         "intpoly_assign",
 444         { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },  { -1, ShenandoahNone },
 445           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 446       };
 447 
 448       if (call->is_call_to_arraycopystub()) {
 449         Node* dest = nullptr;
 450         const TypeTuple* args = n->as_Call()->_tf->domain_sig();
 451         for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
 452           if (args->field_at(i)->isa_ptr()) {
 453             j++;
 454             if (j == 2) {
 455               dest = n->in(i);
 456               break;
 457             }
 458           }
 459         }
 460         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
 461             !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
 462           report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
 463         }
 464       } else if (strlen(call->_name) > 5 &&
 465                  !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
 466         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
 467           report_verify_failure("Shenandoah verification: _fill should have barriers", n);
 468         }
 469       } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
 470         // skip

 551         { { 2, ShenandoahLoad },                  { 3, ShenandoahLoad } },
 552         Op_VectorizedHashCode,
 553         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
 554         Op_EncodeISOArray,
 555         { { 2, ShenandoahLoad },                  { 3, ShenandoahStore } },
 556         Op_CountPositives,
 557         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone} },
 558         Op_CastP2X,
 559         { { 1, ShenandoahLoad },                  { -1, ShenandoahNone} },
 560         Op_StrIndexOfChar,
 561         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
 562       };
 563 
 564       const int others_len = sizeof(others) / sizeof(others[0]);
 565       int i = 0;
 566       for (; i < others_len; i++) {
 567         if (others[i].opcode == n->Opcode()) {
 568           break;
 569         }
 570       }
 571       uint stop = n->is_Call() ? n->as_Call()->tf()->domain_sig()->cnt() : n->req();
 572       if (i != others_len) {
 573         const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
 574         for (uint j = 0; j < inputs_len; j++) {
 575           int pos = others[i].inputs[j].pos;
 576           if (pos == -1) {
 577             break;
 578           }
 579           if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
 580             report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
 581           }
 582         }
 583         for (uint j = 1; j < stop; j++) {
 584           if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
 585               n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
 586             uint k = 0;
 587             for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
 588             if (k == inputs_len) {
 589               fatal("arg %d for node %s not covered", j, n->Name());
 590             }
 591           }

 786     mem_ctrl = phase->ctrl_or_self(mem);
 787   }
 788   return mem;
 789 }
 790 
 791 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
 792   Node* mem = nullptr;
 793   Node* c = ctrl;
 794   do {
 795     if (c->is_Region()) {
 796       for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
 797         Node* u = c->fast_out(i);
 798         if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
 799           if (u->adr_type() == TypePtr::BOTTOM) {
 800             mem = u;
 801           }
 802         }
 803       }
 804     } else {
 805       if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
 806         CallProjections* projs = c->as_Call()->extract_projections(true, false);
 807         if (projs->fallthrough_memproj != nullptr) {
 808           if (projs->fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
 809             if (projs->catchall_memproj == nullptr) {
 810               mem = projs->fallthrough_memproj;

 811             } else {
 812               if (phase->is_dominator(projs->fallthrough_catchproj, ctrl)) {
 813                 mem = projs->fallthrough_memproj;
 814               } else {
 815                 assert(phase->is_dominator(projs->catchall_catchproj, ctrl), "one proj must dominate barrier");
 816                 mem = projs->catchall_memproj;
 817               }
 818             }
 819           }
 820         } else {
 821           Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
 822           if (proj != nullptr &&
 823               proj->adr_type() == TypePtr::BOTTOM) {
 824             mem = proj;
 825           }
 826         }
 827       } else {
 828         for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
 829           Node* u = c->fast_out(i);
 830           if (u->is_Proj() &&
 831               u->bottom_type() == Type::MEMORY &&
 832               u->adr_type() == TypePtr::BOTTOM) {
 833               assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
 834               assert(mem == nullptr, "only one proj");
 835               mem = u;
 836           }

1057       if (old_c != ctrl ||
1058           is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1059           ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1060         phase->igvn().rehash_node_delayed(u);
1061         int nb = u->replace_edge(ctrl, region, &phase->igvn());
1062         if (u->is_CFG()) {
1063           if (phase->idom(u) == ctrl) {
1064             phase->set_idom(u, region, phase->dom_depth(region));
1065           }
1066         } else if (phase->get_ctrl(u) == ctrl) {
1067           assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1068           uses.push(u);
1069         }
1070         assert(nb == 1, "more than 1 ctrl input?");
1071         --i, imax -= nb;
1072       }
1073     }
1074   }
1075 }
1076 
1077 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections* projs, PhaseIdealLoop* phase) {
1078   Node* region = nullptr;
1079   while (c != ctrl) {
1080     if (c->is_Region()) {
1081       region = c;
1082     }
1083     c = phase->idom(c);
1084   }
1085   assert(region != nullptr, "");
1086   Node* phi = new PhiNode(region, n->bottom_type());
1087   for (uint j = 1; j < region->req(); j++) {
1088     Node* in = region->in(j);
1089     if (phase->is_dominator(projs->fallthrough_catchproj, in)) {
1090       phi->init_req(j, n);
1091     } else if (phase->is_dominator(projs->catchall_catchproj, in)) {
1092       phi->init_req(j, n_clone);
1093     } else {
1094       phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1095     }
1096   }
1097   phase->register_new_node(phi, region);
1098   return phi;
1099 }
1100 
1101 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1102   ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1103 
1104   Unique_Node_List uses;
1105   Node_Stack stack(0);
1106   Node_List clones;
1107   for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1108     ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1109 
1110     Node* ctrl = phase->get_ctrl(lrb);
1111     Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);

1174               if (phase->has_ctrl(in)) {
1175                 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1176 #ifdef ASSERT
1177                   for (uint i = 0; i < stack.size(); i++) {
1178                     assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1179                   }
1180 #endif
1181                   stack.push(in, 0);
1182                 }
1183               } else {
1184                 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1185               }
1186             }
1187           } else {
1188             phase->set_ctrl(n, call->in(0));
1189             stack.pop();
1190           }
1191         } while(stack.size() > 0);
1192         continue;
1193       }
1194       CallProjections* projs = call->extract_projections(false, false);

1195 
1196       // If this is a runtime call, it doesn't have an exception handling path
1197       if (projs->fallthrough_catchproj == nullptr) {
1198         assert(call->method() == nullptr, "should be runtime call");
1199         assert(projs->catchall_catchproj == nullptr, "runtime call should not have catch all projection");
1200         continue;
1201       }
1202 
1203       // Otherwise, clone the barrier so there's one for the fallthrough and one for the exception handling path
1204 #ifdef ASSERT
1205       VectorSet cloned;
1206 #endif
1207       Node* lrb_clone = lrb->clone();
1208       phase->register_new_node(lrb_clone, projs->catchall_catchproj);
1209       phase->set_ctrl(lrb, projs->fallthrough_catchproj);
1210 
1211       stack.push(lrb, 0);
1212       clones.push(lrb_clone);
1213 
1214       do {
1215         assert(stack.size() == clones.size(), "");
1216         Node* n = stack.node();
1217 #ifdef ASSERT
1218         if (n->is_Load()) {
1219           Node* mem = n->in(MemNode::Memory);
1220           for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1221             Node* u = mem->fast_out(j);
1222             assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1223           }
1224         }
1225 #endif
1226         uint idx = stack.index();
1227         Node* n_clone = clones.at(clones.size()-1);
1228         if (idx < n->outcnt()) {
1229           Node* u = n->raw_out(idx);
1230           Node* c = phase->ctrl_or_self(u);
1231           if (phase->is_dominator(call, c) && phase->is_dominator(c, projs->fallthrough_proj)) {
1232             stack.set_index(idx+1);
1233             assert(!u->is_CFG(), "");
1234             stack.push(u, 0);
1235             assert(!cloned.test_set(u->_idx), "only one clone");
1236             Node* u_clone = u->clone();
1237             int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1238             assert(nb > 0, "should have replaced some uses");
1239             phase->register_new_node(u_clone, projs->catchall_catchproj);
1240             clones.push(u_clone);
1241             phase->set_ctrl(u, projs->fallthrough_catchproj);
1242           } else {
1243             bool replaced = false;
1244             if (u->is_Phi()) {
1245               for (uint k = 1; k < u->req(); k++) {
1246                 if (u->in(k) == n) {
1247                   if (phase->is_dominator(projs->catchall_catchproj, u->in(0)->in(k))) {
1248                     phase->igvn().replace_input_of(u, k, n_clone);
1249                     replaced = true;
1250                   } else if (!phase->is_dominator(projs->fallthrough_catchproj, u->in(0)->in(k))) {
1251                     phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1252                     replaced = true;
1253                   }
1254                 }
1255               }
1256             } else {
1257               if (phase->is_dominator(projs->catchall_catchproj, c)) {
1258                 phase->igvn().rehash_node_delayed(u);
1259                 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1260                 assert(nb > 0, "should have replaced some uses");
1261                 replaced = true;
1262               } else if (!phase->is_dominator(projs->fallthrough_catchproj, c)) {
1263                 if (u->is_If()) {
1264                   // Can't break If/Bool/Cmp chain
1265                   assert(n->is_Bool(), "unexpected If shape");
1266                   assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1267                   assert(n_clone->is_Bool(), "unexpected clone");
1268                   assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1269                   Node* bol_clone = n->clone();
1270                   Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1271                   bol_clone->set_req(1, cmp_clone);
1272 
1273                   Node* nn = stack.node_at(stack.size()-3);
1274                   Node* nn_clone = clones.at(clones.size()-3);
1275                   assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1276 
1277                   int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1278                                                    &phase->igvn());
1279                   assert(nb > 0, "should have replaced some uses");
1280 
1281                   phase->register_new_node(bol_clone, u->in(0));
1282                   phase->register_new_node(cmp_clone, u->in(0));

1833             } else {
1834               assert(c != c->in(0), "");
1835               c = c->in(0);
1836             }
1837           }
1838         }
1839       }
1840     } while (stack.size() > 0);
1841     assert(mem != nullptr, "should have found safepoint");
1842   } else {
1843     mem = phi_mem;
1844   }
1845   return mem;
1846 }
1847 
1848 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
1849   Node* c = _phase->get_ctrl(n);
1850   if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
1851     assert(c == n->in(0), "");
1852     CallNode* call = c->as_Call();
1853     CallProjections* projs = call->extract_projections(true, false);
1854     if (projs->catchall_memproj != nullptr) {
1855       if (projs->fallthrough_memproj == n) {
1856         c = projs->fallthrough_catchproj;

1857       } else {
1858         assert(projs->catchall_memproj == n, "");
1859         c = projs->catchall_catchproj;
1860       }
1861     }
1862   }
1863   return c;
1864 }
1865 
1866 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
1867   if (_phase->has_ctrl(n))
1868     return get_ctrl(n);
1869   else {
1870     assert (n->is_CFG(), "must be a CFG node");
1871     return n;
1872   }
1873 }
1874 
1875 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
1876   return m != nullptr && get_ctrl(m) == c;
1877 }
1878 
1879 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
< prev index next >