< prev index next >

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

Print this page

 430           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 431         "sha512_implCompressMB",
 432         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
 433           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 434         "encodeBlock",
 435         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 436           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 437         "decodeBlock",
 438         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 439           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 440         "intpoly_montgomeryMult_P256",
 441         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahLoad  },   { TypeFunc::Parms+2, ShenandoahStore },
 442           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 443         "intpoly_assign",
 444         { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },  { -1, ShenandoahNone },
 445           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 446       };
 447 
 448       if (call->is_call_to_arraycopystub()) {
 449         Node* dest = nullptr;
 450         const TypeTuple* args = n->as_Call()->_tf->domain();
 451         for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
 452           if (args->field_at(i)->isa_ptr()) {
 453             j++;
 454             if (j == 2) {
 455               dest = n->in(i);
 456               break;
 457             }
 458           }
 459         }
 460         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
 461             !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
 462           report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
 463         }
 464       } else if (strlen(call->_name) > 5 &&
 465                  !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
 466         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
 467           report_verify_failure("Shenandoah verification: _fill should have barriers", n);
 468         }
 469       } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
 470         // skip

 551         { { 2, ShenandoahLoad },                  { 3, ShenandoahLoad } },
 552         Op_VectorizedHashCode,
 553         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
 554         Op_EncodeISOArray,
 555         { { 2, ShenandoahLoad },                  { 3, ShenandoahStore } },
 556         Op_CountPositives,
 557         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone} },
 558         Op_CastP2X,
 559         { { 1, ShenandoahLoad },                  { -1, ShenandoahNone} },
 560         Op_StrIndexOfChar,
 561         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
 562       };
 563 
 564       const int others_len = sizeof(others) / sizeof(others[0]);
 565       int i = 0;
 566       for (; i < others_len; i++) {
 567         if (others[i].opcode == n->Opcode()) {
 568           break;
 569         }
 570       }
 571       uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
 572       if (i != others_len) {
 573         const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
 574         for (uint j = 0; j < inputs_len; j++) {
 575           int pos = others[i].inputs[j].pos;
 576           if (pos == -1) {
 577             break;
 578           }
 579           if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
 580             report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
 581           }
 582         }
 583         for (uint j = 1; j < stop; j++) {
 584           if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
 585               n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
 586             uint k = 0;
 587             for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
 588             if (k == inputs_len) {
 589               fatal("arg %d for node %s not covered", j, n->Name());
 590             }
 591           }

 771     mem_ctrl = phase->ctrl_or_self(mem);
 772   }
 773   return mem;
 774 }
 775 
 776 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
 777   Node* mem = nullptr;
 778   Node* c = ctrl;
 779   do {
 780     if (c->is_Region()) {
 781       for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
 782         Node* u = c->fast_out(i);
 783         if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
 784           if (u->adr_type() == TypePtr::BOTTOM) {
 785             mem = u;
 786           }
 787         }
 788       }
 789     } else {
 790       if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
 791         CallProjections projs;
 792         c->as_Call()->extract_projections(&projs, true, false);
 793         if (projs.fallthrough_memproj != nullptr) {
 794           if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
 795             if (projs.catchall_memproj == nullptr) {
 796               mem = projs.fallthrough_memproj;
 797             } else {
 798               if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
 799                 mem = projs.fallthrough_memproj;
 800               } else {
 801                 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
 802                 mem = projs.catchall_memproj;
 803               }
 804             }
 805           }
 806         } else {
 807           Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
 808           if (proj != nullptr &&
 809               proj->adr_type() == TypePtr::BOTTOM) {
 810             mem = proj;
 811           }
 812         }
 813       } else {
 814         for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
 815           Node* u = c->fast_out(i);
 816           if (u->is_Proj() &&
 817               u->bottom_type() == Type::MEMORY &&
 818               u->adr_type() == TypePtr::BOTTOM) {
 819               assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
 820               assert(mem == nullptr, "only one proj");
 821               mem = u;
 822           }

1043       if (c != ctrl ||
1044           is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1045           ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1046         phase->igvn().rehash_node_delayed(u);
1047         int nb = u->replace_edge(ctrl, region, &phase->igvn());
1048         if (u->is_CFG()) {
1049           if (phase->idom(u) == ctrl) {
1050             phase->set_idom(u, region, phase->dom_depth(region));
1051           }
1052         } else if (phase->get_ctrl(u) == ctrl) {
1053           assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1054           uses.push(u);
1055         }
1056         assert(nb == 1, "more than 1 ctrl input?");
1057         --i, imax -= nb;
1058       }
1059     }
1060   }
1061 }
1062 
1063 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
1064   Node* region = nullptr;
1065   while (c != ctrl) {
1066     if (c->is_Region()) {
1067       region = c;
1068     }
1069     c = phase->idom(c);
1070   }
1071   assert(region != nullptr, "");
1072   Node* phi = new PhiNode(region, n->bottom_type());
1073   for (uint j = 1; j < region->req(); j++) {
1074     Node* in = region->in(j);
1075     if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1076       phi->init_req(j, n);
1077     } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1078       phi->init_req(j, n_clone);
1079     } else {
1080       phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1081     }
1082   }
1083   phase->register_new_node(phi, region);
1084   return phi;
1085 }
1086 
1087 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1088   ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1089 
1090   Unique_Node_List uses;
1091   Node_Stack stack(0);
1092   Node_List clones;
1093   for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1094     ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1095 
1096     Node* ctrl = phase->get_ctrl(lrb);
1097     Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);

1160               if (phase->has_ctrl(in)) {
1161                 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1162 #ifdef ASSERT
1163                   for (uint i = 0; i < stack.size(); i++) {
1164                     assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1165                   }
1166 #endif
1167                   stack.push(in, 0);
1168                 }
1169               } else {
1170                 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1171               }
1172             }
1173           } else {
1174             phase->set_ctrl(n, call->in(0));
1175             stack.pop();
1176           }
1177         } while(stack.size() > 0);
1178         continue;
1179       }
1180       CallProjections projs;
1181       call->extract_projections(&projs, false, false);
1182 
1183       // If this is a runtime call, it doesn't have an exception handling path
1184       if (projs.fallthrough_catchproj == nullptr) {
1185         assert(call->method() == nullptr, "should be runtime call");
1186         assert(projs.catchall_catchproj == nullptr, "runtime call should not have catch all projection");
1187         continue;
1188       }
1189 
1190       // Otherwise, clone the barrier so there's one for the fallthrough and one for the exception handling path
1191 #ifdef ASSERT
1192       VectorSet cloned;
1193 #endif
1194       Node* lrb_clone = lrb->clone();
1195       phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1196       phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1197 
1198       stack.push(lrb, 0);
1199       clones.push(lrb_clone);
1200 
1201       do {
1202         assert(stack.size() == clones.size(), "");
1203         Node* n = stack.node();
1204 #ifdef ASSERT
1205         if (n->is_Load()) {
1206           Node* mem = n->in(MemNode::Memory);
1207           for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1208             Node* u = mem->fast_out(j);
1209             assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1210           }
1211         }
1212 #endif
1213         uint idx = stack.index();
1214         Node* n_clone = clones.at(clones.size()-1);
1215         if (idx < n->outcnt()) {
1216           Node* u = n->raw_out(idx);
1217           Node* c = phase->ctrl_or_self(u);
1218           if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) {
1219             stack.set_index(idx+1);
1220             assert(!u->is_CFG(), "");
1221             stack.push(u, 0);
1222             assert(!cloned.test_set(u->_idx), "only one clone");
1223             Node* u_clone = u->clone();
1224             int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1225             assert(nb > 0, "should have replaced some uses");
1226             phase->register_new_node(u_clone, projs.catchall_catchproj);
1227             clones.push(u_clone);
1228             phase->set_ctrl(u, projs.fallthrough_catchproj);
1229           } else {
1230             bool replaced = false;
1231             if (u->is_Phi()) {
1232               for (uint k = 1; k < u->req(); k++) {
1233                 if (u->in(k) == n) {
1234                   if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1235                     phase->igvn().replace_input_of(u, k, n_clone);
1236                     replaced = true;
1237                   } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1238                     phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1239                     replaced = true;
1240                   }
1241                 }
1242               }
1243             } else {
1244               if (phase->is_dominator(projs.catchall_catchproj, c)) {
1245                 phase->igvn().rehash_node_delayed(u);
1246                 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1247                 assert(nb > 0, "should have replaced some uses");
1248                 replaced = true;
1249               } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1250                 if (u->is_If()) {
1251                   // Can't break If/Bool/Cmp chain
1252                   assert(n->is_Bool(), "unexpected If shape");
1253                   assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1254                   assert(n_clone->is_Bool(), "unexpected clone");
1255                   assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1256                   Node* bol_clone = n->clone();
1257                   Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1258                   bol_clone->set_req(1, cmp_clone);
1259 
1260                   Node* nn = stack.node_at(stack.size()-3);
1261                   Node* nn_clone = clones.at(clones.size()-3);
1262                   assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1263 
1264                   int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1265                                                    &phase->igvn());
1266                   assert(nb > 0, "should have replaced some uses");
1267 
1268                   phase->register_new_node(bol_clone, u->in(0));
1269                   phase->register_new_node(cmp_clone, u->in(0));

1820             } else {
1821               assert(c != c->in(0), "");
1822               c = c->in(0);
1823             }
1824           }
1825         }
1826       }
1827     } while (stack.size() > 0);
1828     assert(mem != nullptr, "should have found safepoint");
1829   } else {
1830     mem = phi_mem;
1831   }
1832   return mem;
1833 }
1834 
1835 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
1836   Node* c = _phase->get_ctrl(n);
1837   if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
1838     assert(c == n->in(0), "");
1839     CallNode* call = c->as_Call();
1840     CallProjections projs;
1841     call->extract_projections(&projs, true, false);
1842     if (projs.catchall_memproj != nullptr) {
1843       if (projs.fallthrough_memproj == n) {
1844         c = projs.fallthrough_catchproj;
1845       } else {
1846         assert(projs.catchall_memproj == n, "");
1847         c = projs.catchall_catchproj;
1848       }
1849     }
1850   }
1851   return c;
1852 }
1853 
1854 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
1855   if (_phase->has_ctrl(n))
1856     return get_ctrl(n);
1857   else {
1858     assert (n->is_CFG(), "must be a CFG node");
1859     return n;
1860   }
1861 }
1862 
1863 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
1864   return m != nullptr && get_ctrl(m) == c;
1865 }
1866 
1867 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {

 430           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 431         "sha512_implCompressMB",
 432         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
 433           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 434         "encodeBlock",
 435         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 436           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 437         "decodeBlock",
 438         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 439           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 440         "intpoly_montgomeryMult_P256",
 441         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahLoad  },   { TypeFunc::Parms+2, ShenandoahStore },
 442           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 443         "intpoly_assign",
 444         { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },  { -1, ShenandoahNone },
 445           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 446       };
 447 
 448       if (call->is_call_to_arraycopystub()) {
 449         Node* dest = nullptr;
 450         const TypeTuple* args = n->as_Call()->_tf->domain_sig();
 451         for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
 452           if (args->field_at(i)->isa_ptr()) {
 453             j++;
 454             if (j == 2) {
 455               dest = n->in(i);
 456               break;
 457             }
 458           }
 459         }
 460         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
 461             !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
 462           report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
 463         }
 464       } else if (strlen(call->_name) > 5 &&
 465                  !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
 466         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
 467           report_verify_failure("Shenandoah verification: _fill should have barriers", n);
 468         }
 469       } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
 470         // skip

 551         { { 2, ShenandoahLoad },                  { 3, ShenandoahLoad } },
 552         Op_VectorizedHashCode,
 553         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
 554         Op_EncodeISOArray,
 555         { { 2, ShenandoahLoad },                  { 3, ShenandoahStore } },
 556         Op_CountPositives,
 557         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone} },
 558         Op_CastP2X,
 559         { { 1, ShenandoahLoad },                  { -1, ShenandoahNone} },
 560         Op_StrIndexOfChar,
 561         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
 562       };
 563 
 564       const int others_len = sizeof(others) / sizeof(others[0]);
 565       int i = 0;
 566       for (; i < others_len; i++) {
 567         if (others[i].opcode == n->Opcode()) {
 568           break;
 569         }
 570       }
 571       uint stop = n->is_Call() ? n->as_Call()->tf()->domain_sig()->cnt() : n->req();
 572       if (i != others_len) {
 573         const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
 574         for (uint j = 0; j < inputs_len; j++) {
 575           int pos = others[i].inputs[j].pos;
 576           if (pos == -1) {
 577             break;
 578           }
 579           if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
 580             report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
 581           }
 582         }
 583         for (uint j = 1; j < stop; j++) {
 584           if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
 585               n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
 586             uint k = 0;
 587             for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
 588             if (k == inputs_len) {
 589               fatal("arg %d for node %s not covered", j, n->Name());
 590             }
 591           }

 771     mem_ctrl = phase->ctrl_or_self(mem);
 772   }
 773   return mem;
 774 }
 775 
 776 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
 777   Node* mem = nullptr;
 778   Node* c = ctrl;
 779   do {
 780     if (c->is_Region()) {
 781       for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
 782         Node* u = c->fast_out(i);
 783         if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
 784           if (u->adr_type() == TypePtr::BOTTOM) {
 785             mem = u;
 786           }
 787         }
 788       }
 789     } else {
 790       if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
 791         CallProjections* projs = c->as_Call()->extract_projections(true, false);
 792         if (projs->fallthrough_memproj != nullptr) {
 793           if (projs->fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
 794             if (projs->catchall_memproj == nullptr) {
 795               mem = projs->fallthrough_memproj;

 796             } else {
 797               if (phase->is_dominator(projs->fallthrough_catchproj, ctrl)) {
 798                 mem = projs->fallthrough_memproj;
 799               } else {
 800                 assert(phase->is_dominator(projs->catchall_catchproj, ctrl), "one proj must dominate barrier");
 801                 mem = projs->catchall_memproj;
 802               }
 803             }
 804           }
 805         } else {
 806           Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
 807           if (proj != nullptr &&
 808               proj->adr_type() == TypePtr::BOTTOM) {
 809             mem = proj;
 810           }
 811         }
 812       } else {
 813         for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
 814           Node* u = c->fast_out(i);
 815           if (u->is_Proj() &&
 816               u->bottom_type() == Type::MEMORY &&
 817               u->adr_type() == TypePtr::BOTTOM) {
 818               assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
 819               assert(mem == nullptr, "only one proj");
 820               mem = u;
 821           }

1042       if (c != ctrl ||
1043           is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1044           ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1045         phase->igvn().rehash_node_delayed(u);
1046         int nb = u->replace_edge(ctrl, region, &phase->igvn());
1047         if (u->is_CFG()) {
1048           if (phase->idom(u) == ctrl) {
1049             phase->set_idom(u, region, phase->dom_depth(region));
1050           }
1051         } else if (phase->get_ctrl(u) == ctrl) {
1052           assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1053           uses.push(u);
1054         }
1055         assert(nb == 1, "more than 1 ctrl input?");
1056         --i, imax -= nb;
1057       }
1058     }
1059   }
1060 }
1061 
1062 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections* projs, PhaseIdealLoop* phase) {
1063   Node* region = nullptr;
1064   while (c != ctrl) {
1065     if (c->is_Region()) {
1066       region = c;
1067     }
1068     c = phase->idom(c);
1069   }
1070   assert(region != nullptr, "");
1071   Node* phi = new PhiNode(region, n->bottom_type());
1072   for (uint j = 1; j < region->req(); j++) {
1073     Node* in = region->in(j);
1074     if (phase->is_dominator(projs->fallthrough_catchproj, in)) {
1075       phi->init_req(j, n);
1076     } else if (phase->is_dominator(projs->catchall_catchproj, in)) {
1077       phi->init_req(j, n_clone);
1078     } else {
1079       phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1080     }
1081   }
1082   phase->register_new_node(phi, region);
1083   return phi;
1084 }
1085 
1086 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1087   ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1088 
1089   Unique_Node_List uses;
1090   Node_Stack stack(0);
1091   Node_List clones;
1092   for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1093     ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1094 
1095     Node* ctrl = phase->get_ctrl(lrb);
1096     Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);

1159               if (phase->has_ctrl(in)) {
1160                 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1161 #ifdef ASSERT
1162                   for (uint i = 0; i < stack.size(); i++) {
1163                     assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1164                   }
1165 #endif
1166                   stack.push(in, 0);
1167                 }
1168               } else {
1169                 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1170               }
1171             }
1172           } else {
1173             phase->set_ctrl(n, call->in(0));
1174             stack.pop();
1175           }
1176         } while(stack.size() > 0);
1177         continue;
1178       }
1179       CallProjections* projs = call->extract_projections(false, false);

1180 
1181       // If this is a runtime call, it doesn't have an exception handling path
1182       if (projs->fallthrough_catchproj == nullptr) {
1183         assert(call->method() == nullptr, "should be runtime call");
1184         assert(projs->catchall_catchproj == nullptr, "runtime call should not have catch all projection");
1185         continue;
1186       }
1187 
1188       // Otherwise, clone the barrier so there's one for the fallthrough and one for the exception handling path
1189 #ifdef ASSERT
1190       VectorSet cloned;
1191 #endif
1192       Node* lrb_clone = lrb->clone();
1193       phase->register_new_node(lrb_clone, projs->catchall_catchproj);
1194       phase->set_ctrl(lrb, projs->fallthrough_catchproj);
1195 
1196       stack.push(lrb, 0);
1197       clones.push(lrb_clone);
1198 
1199       do {
1200         assert(stack.size() == clones.size(), "");
1201         Node* n = stack.node();
1202 #ifdef ASSERT
1203         if (n->is_Load()) {
1204           Node* mem = n->in(MemNode::Memory);
1205           for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1206             Node* u = mem->fast_out(j);
1207             assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1208           }
1209         }
1210 #endif
1211         uint idx = stack.index();
1212         Node* n_clone = clones.at(clones.size()-1);
1213         if (idx < n->outcnt()) {
1214           Node* u = n->raw_out(idx);
1215           Node* c = phase->ctrl_or_self(u);
1216           if (phase->is_dominator(call, c) && phase->is_dominator(c, projs->fallthrough_proj)) {
1217             stack.set_index(idx+1);
1218             assert(!u->is_CFG(), "");
1219             stack.push(u, 0);
1220             assert(!cloned.test_set(u->_idx), "only one clone");
1221             Node* u_clone = u->clone();
1222             int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1223             assert(nb > 0, "should have replaced some uses");
1224             phase->register_new_node(u_clone, projs->catchall_catchproj);
1225             clones.push(u_clone);
1226             phase->set_ctrl(u, projs->fallthrough_catchproj);
1227           } else {
1228             bool replaced = false;
1229             if (u->is_Phi()) {
1230               for (uint k = 1; k < u->req(); k++) {
1231                 if (u->in(k) == n) {
1232                   if (phase->is_dominator(projs->catchall_catchproj, u->in(0)->in(k))) {
1233                     phase->igvn().replace_input_of(u, k, n_clone);
1234                     replaced = true;
1235                   } else if (!phase->is_dominator(projs->fallthrough_catchproj, u->in(0)->in(k))) {
1236                     phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1237                     replaced = true;
1238                   }
1239                 }
1240               }
1241             } else {
1242               if (phase->is_dominator(projs->catchall_catchproj, c)) {
1243                 phase->igvn().rehash_node_delayed(u);
1244                 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1245                 assert(nb > 0, "should have replaced some uses");
1246                 replaced = true;
1247               } else if (!phase->is_dominator(projs->fallthrough_catchproj, c)) {
1248                 if (u->is_If()) {
1249                   // Can't break If/Bool/Cmp chain
1250                   assert(n->is_Bool(), "unexpected If shape");
1251                   assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1252                   assert(n_clone->is_Bool(), "unexpected clone");
1253                   assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1254                   Node* bol_clone = n->clone();
1255                   Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1256                   bol_clone->set_req(1, cmp_clone);
1257 
1258                   Node* nn = stack.node_at(stack.size()-3);
1259                   Node* nn_clone = clones.at(clones.size()-3);
1260                   assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1261 
1262                   int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1263                                                    &phase->igvn());
1264                   assert(nb > 0, "should have replaced some uses");
1265 
1266                   phase->register_new_node(bol_clone, u->in(0));
1267                   phase->register_new_node(cmp_clone, u->in(0));

1818             } else {
1819               assert(c != c->in(0), "");
1820               c = c->in(0);
1821             }
1822           }
1823         }
1824       }
1825     } while (stack.size() > 0);
1826     assert(mem != nullptr, "should have found safepoint");
1827   } else {
1828     mem = phi_mem;
1829   }
1830   return mem;
1831 }
1832 
1833 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
1834   Node* c = _phase->get_ctrl(n);
1835   if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
1836     assert(c == n->in(0), "");
1837     CallNode* call = c->as_Call();
1838     CallProjections* projs = call->extract_projections(true, false);
1839     if (projs->catchall_memproj != nullptr) {
1840       if (projs->fallthrough_memproj == n) {
1841         c = projs->fallthrough_catchproj;

1842       } else {
1843         assert(projs->catchall_memproj == n, "");
1844         c = projs->catchall_catchproj;
1845       }
1846     }
1847   }
1848   return c;
1849 }
1850 
1851 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
1852   if (_phase->has_ctrl(n))
1853     return get_ctrl(n);
1854   else {
1855     assert (n->is_CFG(), "must be a CFG node");
1856     return n;
1857   }
1858 }
1859 
1860 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
1861   return m != nullptr && get_ctrl(m) == c;
1862 }
1863 
1864 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
< prev index next >