< prev index next >

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

Print this page

 446           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 447         "sha1_implCompressMB",
 448         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
 449           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 450         "sha256_implCompressMB",
 451         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
 452           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 453         "sha512_implCompressMB",
 454         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
 455           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 456         "encodeBlock",
 457         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 458           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 459         "decodeBlock",
 460         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 461           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 462       };
 463 
 464       if (call->is_call_to_arraycopystub()) {
 465         Node* dest = NULL;
 466         const TypeTuple* args = n->as_Call()->_tf->domain();
 467         for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
 468           if (args->field_at(i)->isa_ptr()) {
 469             j++;
 470             if (j == 2) {
 471               dest = n->in(i);
 472               break;
 473             }
 474           }
 475         }
 476         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
 477             !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
 478           report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
 479         }
 480       } else if (strlen(call->_name) > 5 &&
 481                  !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
 482         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
 483           report_verify_failure("Shenandoah verification: _fill should have barriers", n);
 484         }
 485       } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
 486         // skip

 565         { { 2, ShenandoahLoad },                  { 4, ShenandoahLoad } },
 566         Op_StrEquals,
 567         { { 2, ShenandoahLoad },                  { 3, ShenandoahLoad } },
 568         Op_EncodeISOArray,
 569         { { 2, ShenandoahLoad },                  { 3, ShenandoahStore } },
 570         Op_HasNegatives,
 571         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone} },
 572         Op_CastP2X,
 573         { { 1, ShenandoahLoad },                  { -1, ShenandoahNone} },
 574         Op_StrIndexOfChar,
 575         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
 576       };
 577 
 578       const int others_len = sizeof(others) / sizeof(others[0]);
 579       int i = 0;
 580       for (; i < others_len; i++) {
 581         if (others[i].opcode == n->Opcode()) {
 582           break;
 583         }
 584       }
 585       uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
 586       if (i != others_len) {
 587         const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
 588         for (uint j = 0; j < inputs_len; j++) {
 589           int pos = others[i].inputs[j].pos;
 590           if (pos == -1) {
 591             break;
 592           }
 593           if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
 594             report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
 595           }
 596         }
 597         for (uint j = 1; j < stop; j++) {
 598           if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
 599               n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
 600             uint k = 0;
 601             for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
 602             if (k == inputs_len) {
 603               fatal("arg %d for node %s not covered", j, n->Name());
 604             }
 605           }

 785     mem_ctrl = phase->ctrl_or_self(mem);
 786   }
 787   return mem;
 788 }
 789 
 790 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
 791   Node* mem = NULL;
 792   Node* c = ctrl;
 793   do {
 794     if (c->is_Region()) {
 795       for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) {
 796         Node* u = c->fast_out(i);
 797         if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
 798           if (u->adr_type() == TypePtr::BOTTOM) {
 799             mem = u;
 800           }
 801         }
 802       }
 803     } else {
 804       if (c->is_Call() && c->as_Call()->adr_type() != NULL) {
 805         CallProjections projs;
 806         c->as_Call()->extract_projections(&projs, true, false);
 807         if (projs.fallthrough_memproj != NULL) {
 808           if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
 809             if (projs.catchall_memproj == NULL) {
 810               mem = projs.fallthrough_memproj;
 811             } else {
 812               if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
 813                 mem = projs.fallthrough_memproj;
 814               } else {
 815                 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
 816                 mem = projs.catchall_memproj;
 817               }
 818             }
 819           }
 820         } else {
 821           Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
 822           if (proj != NULL &&
 823               proj->adr_type() == TypePtr::BOTTOM) {
 824             mem = proj;
 825           }
 826         }
 827       } else {
 828         for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
 829           Node* u = c->fast_out(i);
 830           if (u->is_Proj() &&
 831               u->bottom_type() == Type::MEMORY &&
 832               u->adr_type() == TypePtr::BOTTOM) {
 833               assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
 834               assert(mem == NULL, "only one proj");
 835               mem = u;
 836           }

1065       if (c != ctrl ||
1066           is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1067           ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1068         phase->igvn().rehash_node_delayed(u);
1069         int nb = u->replace_edge(ctrl, region, &phase->igvn());
1070         if (u->is_CFG()) {
1071           if (phase->idom(u) == ctrl) {
1072             phase->set_idom(u, region, phase->dom_depth(region));
1073           }
1074         } else if (phase->get_ctrl(u) == ctrl) {
1075           assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1076           uses.push(u);
1077         }
1078         assert(nb == 1, "more than 1 ctrl input?");
1079         --i, imax -= nb;
1080       }
1081     }
1082   }
1083 }
1084 
1085 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
1086   Node* region = NULL;
1087   while (c != ctrl) {
1088     if (c->is_Region()) {
1089       region = c;
1090     }
1091     c = phase->idom(c);
1092   }
1093   assert(region != NULL, "");
1094   Node* phi = new PhiNode(region, n->bottom_type());
1095   for (uint j = 1; j < region->req(); j++) {
1096     Node* in = region->in(j);
1097     if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1098       phi->init_req(j, n);
1099     } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1100       phi->init_req(j, n_clone);
1101     } else {
1102       phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1103     }
1104   }
1105   phase->register_new_node(phi, region);
1106   return phi;
1107 }
1108 
1109 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1110   ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1111 
1112   Unique_Node_List uses;
1113   for (int i = 0; i < state->iu_barriers_count(); i++) {
1114     Node* barrier = state->iu_barrier(i);
1115     Node* ctrl = phase->get_ctrl(barrier);
1116     IdealLoopTree* loop = phase->get_loop(ctrl);
1117     Node* head = loop->head();
1118     if (head->is_OuterStripMinedLoop()) {
1119       // Expanding a barrier here will break loop strip mining

1195               if (phase->has_ctrl(in)) {
1196                 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1197 #ifdef ASSERT
1198                   for (uint i = 0; i < stack.size(); i++) {
1199                     assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1200                   }
1201 #endif
1202                   stack.push(in, 0);
1203                 }
1204               } else {
1205                 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1206               }
1207             }
1208           } else {
1209             phase->set_ctrl(n, call->in(0));
1210             stack.pop();
1211           }
1212         } while(stack.size() > 0);
1213         continue;
1214       }
1215       CallProjections projs;
1216       call->extract_projections(&projs, false, false);
1217 
1218 #ifdef ASSERT
1219       VectorSet cloned;
1220 #endif
1221       Node* lrb_clone = lrb->clone();
1222       phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1223       phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1224 
1225       stack.push(lrb, 0);
1226       clones.push(lrb_clone);
1227 
1228       do {
1229         assert(stack.size() == clones.size(), "");
1230         Node* n = stack.node();
1231 #ifdef ASSERT
1232         if (n->is_Load()) {
1233           Node* mem = n->in(MemNode::Memory);
1234           for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1235             Node* u = mem->fast_out(j);
1236             assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1237           }
1238         }
1239 #endif
1240         uint idx = stack.index();
1241         Node* n_clone = clones.at(clones.size()-1);
1242         if (idx < n->outcnt()) {
1243           Node* u = n->raw_out(idx);
1244           Node* c = phase->ctrl_or_self(u);
1245           if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) {
1246             stack.set_index(idx+1);
1247             assert(!u->is_CFG(), "");
1248             stack.push(u, 0);
1249             assert(!cloned.test_set(u->_idx), "only one clone");
1250             Node* u_clone = u->clone();
1251             int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1252             assert(nb > 0, "should have replaced some uses");
1253             phase->register_new_node(u_clone, projs.catchall_catchproj);
1254             clones.push(u_clone);
1255             phase->set_ctrl(u, projs.fallthrough_catchproj);
1256           } else {
1257             bool replaced = false;
1258             if (u->is_Phi()) {
1259               for (uint k = 1; k < u->req(); k++) {
1260                 if (u->in(k) == n) {
1261                   if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1262                     phase->igvn().replace_input_of(u, k, n_clone);
1263                     replaced = true;
1264                   } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1265                     phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1266                     replaced = true;
1267                   }
1268                 }
1269               }
1270             } else {
1271               if (phase->is_dominator(projs.catchall_catchproj, c)) {
1272                 phase->igvn().rehash_node_delayed(u);
1273                 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1274                 assert(nb > 0, "should have replaced some uses");
1275                 replaced = true;
1276               } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1277                 if (u->is_If()) {
1278                   // Can't break If/Bool/Cmp chain
1279                   assert(n->is_Bool(), "unexpected If shape");
1280                   assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1281                   assert(n_clone->is_Bool(), "unexpected clone");
1282                   assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1283                   Node* bol_clone = n->clone();
1284                   Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1285                   bol_clone->set_req(1, cmp_clone);
1286 
1287                   Node* nn = stack.node_at(stack.size()-3);
1288                   Node* nn_clone = clones.at(clones.size()-3);
1289                   assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1290 
1291                   int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1292                                                    &phase->igvn());
1293                   assert(nb > 0, "should have replaced some uses");
1294 
1295                   phase->register_new_node(bol_clone, u->in(0));
1296                   phase->register_new_node(cmp_clone, u->in(0));

2443             } else {
2444               assert(c != c->in(0), "");
2445               c = c->in(0);
2446             }
2447           }
2448         }
2449       }
2450     } while (stack.size() > 0);
2451     assert(mem != NULL, "should have found safepoint");
2452   } else {
2453     mem = phi_mem;
2454   }
2455   return mem;
2456 }
2457 
2458 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
2459   Node* c = _phase->get_ctrl(n);
2460   if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Call()) {
2461     assert(c == n->in(0), "");
2462     CallNode* call = c->as_Call();
2463     CallProjections projs;
2464     call->extract_projections(&projs, true, false);
2465     if (projs.catchall_memproj != NULL) {
2466       if (projs.fallthrough_memproj == n) {
2467         c = projs.fallthrough_catchproj;
2468       } else {
2469         assert(projs.catchall_memproj == n, "");
2470         c = projs.catchall_catchproj;
2471       }
2472     }
2473   }
2474   return c;
2475 }
2476 
2477 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
2478   if (_phase->has_ctrl(n))
2479     return get_ctrl(n);
2480   else {
2481     assert (n->is_CFG(), "must be a CFG node");
2482     return n;
2483   }
2484 }
2485 
2486 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2487   return m != NULL && get_ctrl(m) == c;
2488 }
2489 
2490 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {

 446           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 447         "sha1_implCompressMB",
 448         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
 449           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 450         "sha256_implCompressMB",
 451         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
 452           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 453         "sha512_implCompressMB",
 454         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },
 455           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 456         "encodeBlock",
 457         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 458           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 459         "decodeBlock",
 460         { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },
 461           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },
 462       };
 463 
 464       if (call->is_call_to_arraycopystub()) {
 465         Node* dest = NULL;
 466         const TypeTuple* args = n->as_Call()->_tf->domain_sig();
 467         for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
 468           if (args->field_at(i)->isa_ptr()) {
 469             j++;
 470             if (j == 2) {
 471               dest = n->in(i);
 472               break;
 473             }
 474           }
 475         }
 476         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
 477             !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
 478           report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
 479         }
 480       } else if (strlen(call->_name) > 5 &&
 481                  !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
 482         if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
 483           report_verify_failure("Shenandoah verification: _fill should have barriers", n);
 484         }
 485       } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
 486         // skip

 565         { { 2, ShenandoahLoad },                  { 4, ShenandoahLoad } },
 566         Op_StrEquals,
 567         { { 2, ShenandoahLoad },                  { 3, ShenandoahLoad } },
 568         Op_EncodeISOArray,
 569         { { 2, ShenandoahLoad },                  { 3, ShenandoahStore } },
 570         Op_HasNegatives,
 571         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone} },
 572         Op_CastP2X,
 573         { { 1, ShenandoahLoad },                  { -1, ShenandoahNone} },
 574         Op_StrIndexOfChar,
 575         { { 2, ShenandoahLoad },                  { -1, ShenandoahNone } },
 576       };
 577 
 578       const int others_len = sizeof(others) / sizeof(others[0]);
 579       int i = 0;
 580       for (; i < others_len; i++) {
 581         if (others[i].opcode == n->Opcode()) {
 582           break;
 583         }
 584       }
 585       uint stop = n->is_Call() ? n->as_Call()->tf()->domain_sig()->cnt() : n->req();
 586       if (i != others_len) {
 587         const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
 588         for (uint j = 0; j < inputs_len; j++) {
 589           int pos = others[i].inputs[j].pos;
 590           if (pos == -1) {
 591             break;
 592           }
 593           if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
 594             report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
 595           }
 596         }
 597         for (uint j = 1; j < stop; j++) {
 598           if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
 599               n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
 600             uint k = 0;
 601             for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
 602             if (k == inputs_len) {
 603               fatal("arg %d for node %s not covered", j, n->Name());
 604             }
 605           }

 785     mem_ctrl = phase->ctrl_or_self(mem);
 786   }
 787   return mem;
 788 }
 789 
 790 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
 791   Node* mem = NULL;
 792   Node* c = ctrl;
 793   do {
 794     if (c->is_Region()) {
 795       for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) {
 796         Node* u = c->fast_out(i);
 797         if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
 798           if (u->adr_type() == TypePtr::BOTTOM) {
 799             mem = u;
 800           }
 801         }
 802       }
 803     } else {
 804       if (c->is_Call() && c->as_Call()->adr_type() != NULL) {
 805         CallProjections* projs = c->as_Call()->extract_projections(true, false);
 806         if (projs->fallthrough_memproj != NULL) {
 807           if (projs->fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
 808             if (projs->catchall_memproj == NULL) {
 809               mem = projs->fallthrough_memproj;

 810             } else {
 811               if (phase->is_dominator(projs->fallthrough_catchproj, ctrl)) {
 812                 mem = projs->fallthrough_memproj;
 813               } else {
 814                 assert(phase->is_dominator(projs->catchall_catchproj, ctrl), "one proj must dominate barrier");
 815                 mem = projs->catchall_memproj;
 816               }
 817             }
 818           }
 819         } else {
 820           Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
 821           if (proj != NULL &&
 822               proj->adr_type() == TypePtr::BOTTOM) {
 823             mem = proj;
 824           }
 825         }
 826       } else {
 827         for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
 828           Node* u = c->fast_out(i);
 829           if (u->is_Proj() &&
 830               u->bottom_type() == Type::MEMORY &&
 831               u->adr_type() == TypePtr::BOTTOM) {
 832               assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
 833               assert(mem == NULL, "only one proj");
 834               mem = u;
 835           }

1064       if (c != ctrl ||
1065           is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1066           ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1067         phase->igvn().rehash_node_delayed(u);
1068         int nb = u->replace_edge(ctrl, region, &phase->igvn());
1069         if (u->is_CFG()) {
1070           if (phase->idom(u) == ctrl) {
1071             phase->set_idom(u, region, phase->dom_depth(region));
1072           }
1073         } else if (phase->get_ctrl(u) == ctrl) {
1074           assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1075           uses.push(u);
1076         }
1077         assert(nb == 1, "more than 1 ctrl input?");
1078         --i, imax -= nb;
1079       }
1080     }
1081   }
1082 }
1083 
1084 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections* projs, PhaseIdealLoop* phase) {
1085   Node* region = NULL;
1086   while (c != ctrl) {
1087     if (c->is_Region()) {
1088       region = c;
1089     }
1090     c = phase->idom(c);
1091   }
1092   assert(region != NULL, "");
1093   Node* phi = new PhiNode(region, n->bottom_type());
1094   for (uint j = 1; j < region->req(); j++) {
1095     Node* in = region->in(j);
1096     if (phase->is_dominator(projs->fallthrough_catchproj, in)) {
1097       phi->init_req(j, n);
1098     } else if (phase->is_dominator(projs->catchall_catchproj, in)) {
1099       phi->init_req(j, n_clone);
1100     } else {
1101       phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1102     }
1103   }
1104   phase->register_new_node(phi, region);
1105   return phi;
1106 }
1107 
1108 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1109   ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1110 
1111   Unique_Node_List uses;
1112   for (int i = 0; i < state->iu_barriers_count(); i++) {
1113     Node* barrier = state->iu_barrier(i);
1114     Node* ctrl = phase->get_ctrl(barrier);
1115     IdealLoopTree* loop = phase->get_loop(ctrl);
1116     Node* head = loop->head();
1117     if (head->is_OuterStripMinedLoop()) {
1118       // Expanding a barrier here will break loop strip mining

1194               if (phase->has_ctrl(in)) {
1195                 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1196 #ifdef ASSERT
1197                   for (uint i = 0; i < stack.size(); i++) {
1198                     assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1199                   }
1200 #endif
1201                   stack.push(in, 0);
1202                 }
1203               } else {
1204                 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1205               }
1206             }
1207           } else {
1208             phase->set_ctrl(n, call->in(0));
1209             stack.pop();
1210           }
1211         } while(stack.size() > 0);
1212         continue;
1213       }
1214       CallProjections* projs = call->extract_projections(false, false);


1215 #ifdef ASSERT
1216       VectorSet cloned;
1217 #endif
1218       Node* lrb_clone = lrb->clone();
1219       phase->register_new_node(lrb_clone, projs->catchall_catchproj);
1220       phase->set_ctrl(lrb, projs->fallthrough_catchproj);
1221 
1222       stack.push(lrb, 0);
1223       clones.push(lrb_clone);
1224 
1225       do {
1226         assert(stack.size() == clones.size(), "");
1227         Node* n = stack.node();
1228 #ifdef ASSERT
1229         if (n->is_Load()) {
1230           Node* mem = n->in(MemNode::Memory);
1231           for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1232             Node* u = mem->fast_out(j);
1233             assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1234           }
1235         }
1236 #endif
1237         uint idx = stack.index();
1238         Node* n_clone = clones.at(clones.size()-1);
1239         if (idx < n->outcnt()) {
1240           Node* u = n->raw_out(idx);
1241           Node* c = phase->ctrl_or_self(u);
1242           if (phase->is_dominator(call, c) && phase->is_dominator(c, projs->fallthrough_proj)) {
1243             stack.set_index(idx+1);
1244             assert(!u->is_CFG(), "");
1245             stack.push(u, 0);
1246             assert(!cloned.test_set(u->_idx), "only one clone");
1247             Node* u_clone = u->clone();
1248             int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1249             assert(nb > 0, "should have replaced some uses");
1250             phase->register_new_node(u_clone, projs->catchall_catchproj);
1251             clones.push(u_clone);
1252             phase->set_ctrl(u, projs->fallthrough_catchproj);
1253           } else {
1254             bool replaced = false;
1255             if (u->is_Phi()) {
1256               for (uint k = 1; k < u->req(); k++) {
1257                 if (u->in(k) == n) {
1258                   if (phase->is_dominator(projs->catchall_catchproj, u->in(0)->in(k))) {
1259                     phase->igvn().replace_input_of(u, k, n_clone);
1260                     replaced = true;
1261                   } else if (!phase->is_dominator(projs->fallthrough_catchproj, u->in(0)->in(k))) {
1262                     phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1263                     replaced = true;
1264                   }
1265                 }
1266               }
1267             } else {
1268               if (phase->is_dominator(projs->catchall_catchproj, c)) {
1269                 phase->igvn().rehash_node_delayed(u);
1270                 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1271                 assert(nb > 0, "should have replaced some uses");
1272                 replaced = true;
1273               } else if (!phase->is_dominator(projs->fallthrough_catchproj, c)) {
1274                 if (u->is_If()) {
1275                   // Can't break If/Bool/Cmp chain
1276                   assert(n->is_Bool(), "unexpected If shape");
1277                   assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1278                   assert(n_clone->is_Bool(), "unexpected clone");
1279                   assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1280                   Node* bol_clone = n->clone();
1281                   Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1282                   bol_clone->set_req(1, cmp_clone);
1283 
1284                   Node* nn = stack.node_at(stack.size()-3);
1285                   Node* nn_clone = clones.at(clones.size()-3);
1286                   assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1287 
1288                   int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1289                                                    &phase->igvn());
1290                   assert(nb > 0, "should have replaced some uses");
1291 
1292                   phase->register_new_node(bol_clone, u->in(0));
1293                   phase->register_new_node(cmp_clone, u->in(0));

2440             } else {
2441               assert(c != c->in(0), "");
2442               c = c->in(0);
2443             }
2444           }
2445         }
2446       }
2447     } while (stack.size() > 0);
2448     assert(mem != NULL, "should have found safepoint");
2449   } else {
2450     mem = phi_mem;
2451   }
2452   return mem;
2453 }
2454 
2455 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
2456   Node* c = _phase->get_ctrl(n);
2457   if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Call()) {
2458     assert(c == n->in(0), "");
2459     CallNode* call = c->as_Call();
2460     CallProjections* projs = call->extract_projections(true, false);
2461     if (projs->catchall_memproj != NULL) {
2462       if (projs->fallthrough_memproj == n) {
2463         c = projs->fallthrough_catchproj;

2464       } else {
2465         assert(projs->catchall_memproj == n, "");
2466         c = projs->catchall_catchproj;
2467       }
2468     }
2469   }
2470   return c;
2471 }
2472 
2473 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
2474   if (_phase->has_ctrl(n))
2475     return get_ctrl(n);
2476   else {
2477     assert (n->is_CFG(), "must be a CFG node");
2478     return n;
2479   }
2480 }
2481 
2482 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2483   return m != NULL && get_ctrl(m) == c;
2484 }
2485 
2486 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
< prev index next >