450 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
451 "sha1_implCompressMB",
452 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
453 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
454 "sha256_implCompressMB",
455 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
456 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
457 "sha512_implCompressMB",
458 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
459 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
460 "encodeBlock",
461 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
462 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
463 "decodeBlock",
464 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
465 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
466 };
467
468 if (call->is_call_to_arraycopystub()) {
469 Node* dest = nullptr;
470 const TypeTuple* args = n->as_Call()->_tf->domain();
471 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
472 if (args->field_at(i)->isa_ptr()) {
473 j++;
474 if (j == 2) {
475 dest = n->in(i);
476 break;
477 }
478 }
479 }
480 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
481 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
482 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
483 }
484 } else if (strlen(call->_name) > 5 &&
485 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
486 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
487 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
488 }
489 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
490 // skip
571 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
572 Op_VectorizedHashCode,
573 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
574 Op_EncodeISOArray,
575 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
576 Op_CountPositives,
577 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
578 Op_CastP2X,
579 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
580 Op_StrIndexOfChar,
581 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
582 };
583
584 const int others_len = sizeof(others) / sizeof(others[0]);
585 int i = 0;
586 for (; i < others_len; i++) {
587 if (others[i].opcode == n->Opcode()) {
588 break;
589 }
590 }
591 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
592 if (i != others_len) {
593 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
594 for (uint j = 0; j < inputs_len; j++) {
595 int pos = others[i].inputs[j].pos;
596 if (pos == -1) {
597 break;
598 }
599 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
600 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
601 }
602 }
603 for (uint j = 1; j < stop; j++) {
604 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
605 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
606 uint k = 0;
607 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
608 if (k == inputs_len) {
609 fatal("arg %d for node %s not covered", j, n->Name());
610 }
611 }
791 mem_ctrl = phase->ctrl_or_self(mem);
792 }
793 return mem;
794 }
795
796 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
797 Node* mem = nullptr;
798 Node* c = ctrl;
799 do {
800 if (c->is_Region()) {
801 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
802 Node* u = c->fast_out(i);
803 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
804 if (u->adr_type() == TypePtr::BOTTOM) {
805 mem = u;
806 }
807 }
808 }
809 } else {
810 if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
811 CallProjections projs;
812 c->as_Call()->extract_projections(&projs, true, false);
813 if (projs.fallthrough_memproj != nullptr) {
814 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
815 if (projs.catchall_memproj == nullptr) {
816 mem = projs.fallthrough_memproj;
817 } else {
818 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
819 mem = projs.fallthrough_memproj;
820 } else {
821 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
822 mem = projs.catchall_memproj;
823 }
824 }
825 }
826 } else {
827 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
828 if (proj != nullptr &&
829 proj->adr_type() == TypePtr::BOTTOM) {
830 mem = proj;
831 }
832 }
833 } else {
834 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
835 Node* u = c->fast_out(i);
836 if (u->is_Proj() &&
837 u->bottom_type() == Type::MEMORY &&
838 u->adr_type() == TypePtr::BOTTOM) {
839 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
840 assert(mem == nullptr, "only one proj");
841 mem = u;
842 }
1062 if (c != ctrl ||
1063 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1064 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1065 phase->igvn().rehash_node_delayed(u);
1066 int nb = u->replace_edge(ctrl, region, &phase->igvn());
1067 if (u->is_CFG()) {
1068 if (phase->idom(u) == ctrl) {
1069 phase->set_idom(u, region, phase->dom_depth(region));
1070 }
1071 } else if (phase->get_ctrl(u) == ctrl) {
1072 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1073 uses.push(u);
1074 }
1075 assert(nb == 1, "more than 1 ctrl input?");
1076 --i, imax -= nb;
1077 }
1078 }
1079 }
1080 }
1081
1082 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
1083 Node* region = nullptr;
1084 while (c != ctrl) {
1085 if (c->is_Region()) {
1086 region = c;
1087 }
1088 c = phase->idom(c);
1089 }
1090 assert(region != nullptr, "");
1091 Node* phi = new PhiNode(region, n->bottom_type());
1092 for (uint j = 1; j < region->req(); j++) {
1093 Node* in = region->in(j);
1094 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1095 phi->init_req(j, n);
1096 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1097 phi->init_req(j, n_clone);
1098 } else {
1099 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1100 }
1101 }
1102 phase->register_new_node(phi, region);
1103 return phi;
1104 }
1105
1106 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1107 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1108
1109 Unique_Node_List uses;
1110 for (int i = 0; i < state->iu_barriers_count(); i++) {
1111 Node* barrier = state->iu_barrier(i);
1112 Node* ctrl = phase->get_ctrl(barrier);
1113 IdealLoopTree* loop = phase->get_loop(ctrl);
1114 Node* head = loop->head();
1115 if (head->is_OuterStripMinedLoop()) {
1116 // Expanding a barrier here will break loop strip mining
1192 if (phase->has_ctrl(in)) {
1193 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1194 #ifdef ASSERT
1195 for (uint i = 0; i < stack.size(); i++) {
1196 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1197 }
1198 #endif
1199 stack.push(in, 0);
1200 }
1201 } else {
1202 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1203 }
1204 }
1205 } else {
1206 phase->set_ctrl(n, call->in(0));
1207 stack.pop();
1208 }
1209 } while(stack.size() > 0);
1210 continue;
1211 }
1212 CallProjections projs;
1213 call->extract_projections(&projs, false, false);
1214
1215 #ifdef ASSERT
1216 VectorSet cloned;
1217 #endif
1218 Node* lrb_clone = lrb->clone();
1219 phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1220 phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1221
1222 stack.push(lrb, 0);
1223 clones.push(lrb_clone);
1224
1225 do {
1226 assert(stack.size() == clones.size(), "");
1227 Node* n = stack.node();
1228 #ifdef ASSERT
1229 if (n->is_Load()) {
1230 Node* mem = n->in(MemNode::Memory);
1231 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1232 Node* u = mem->fast_out(j);
1233 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1234 }
1235 }
1236 #endif
1237 uint idx = stack.index();
1238 Node* n_clone = clones.at(clones.size()-1);
1239 if (idx < n->outcnt()) {
1240 Node* u = n->raw_out(idx);
1241 Node* c = phase->ctrl_or_self(u);
1242 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) {
1243 stack.set_index(idx+1);
1244 assert(!u->is_CFG(), "");
1245 stack.push(u, 0);
1246 assert(!cloned.test_set(u->_idx), "only one clone");
1247 Node* u_clone = u->clone();
1248 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1249 assert(nb > 0, "should have replaced some uses");
1250 phase->register_new_node(u_clone, projs.catchall_catchproj);
1251 clones.push(u_clone);
1252 phase->set_ctrl(u, projs.fallthrough_catchproj);
1253 } else {
1254 bool replaced = false;
1255 if (u->is_Phi()) {
1256 for (uint k = 1; k < u->req(); k++) {
1257 if (u->in(k) == n) {
1258 if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1259 phase->igvn().replace_input_of(u, k, n_clone);
1260 replaced = true;
1261 } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1262 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1263 replaced = true;
1264 }
1265 }
1266 }
1267 } else {
1268 if (phase->is_dominator(projs.catchall_catchproj, c)) {
1269 phase->igvn().rehash_node_delayed(u);
1270 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1271 assert(nb > 0, "should have replaced some uses");
1272 replaced = true;
1273 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1274 if (u->is_If()) {
1275 // Can't break If/Bool/Cmp chain
1276 assert(n->is_Bool(), "unexpected If shape");
1277 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1278 assert(n_clone->is_Bool(), "unexpected clone");
1279 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1280 Node* bol_clone = n->clone();
1281 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1282 bol_clone->set_req(1, cmp_clone);
1283
1284 Node* nn = stack.node_at(stack.size()-3);
1285 Node* nn_clone = clones.at(clones.size()-3);
1286 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1287
1288 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1289 &phase->igvn());
1290 assert(nb > 0, "should have replaced some uses");
1291
1292 phase->register_new_node(bol_clone, u->in(0));
1293 phase->register_new_node(cmp_clone, u->in(0));
2324 } else {
2325 assert(c != c->in(0), "");
2326 c = c->in(0);
2327 }
2328 }
2329 }
2330 }
2331 } while (stack.size() > 0);
2332 assert(mem != nullptr, "should have found safepoint");
2333 } else {
2334 mem = phi_mem;
2335 }
2336 return mem;
2337 }
2338
2339 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
2340 Node* c = _phase->get_ctrl(n);
2341 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
2342 assert(c == n->in(0), "");
2343 CallNode* call = c->as_Call();
2344 CallProjections projs;
2345 call->extract_projections(&projs, true, false);
2346 if (projs.catchall_memproj != nullptr) {
2347 if (projs.fallthrough_memproj == n) {
2348 c = projs.fallthrough_catchproj;
2349 } else {
2350 assert(projs.catchall_memproj == n, "");
2351 c = projs.catchall_catchproj;
2352 }
2353 }
2354 }
2355 return c;
2356 }
2357
2358 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
2359 if (_phase->has_ctrl(n))
2360 return get_ctrl(n);
2361 else {
2362 assert (n->is_CFG(), "must be a CFG node");
2363 return n;
2364 }
2365 }
2366
2367 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2368 return m != nullptr && get_ctrl(m) == c;
2369 }
2370
2371 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|
450 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
451 "sha1_implCompressMB",
452 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
453 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
454 "sha256_implCompressMB",
455 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
456 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
457 "sha512_implCompressMB",
458 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
459 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
460 "encodeBlock",
461 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
462 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
463 "decodeBlock",
464 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
465 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
466 };
467
468 if (call->is_call_to_arraycopystub()) {
469 Node* dest = nullptr;
470 const TypeTuple* args = n->as_Call()->_tf->domain_sig();
471 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
472 if (args->field_at(i)->isa_ptr()) {
473 j++;
474 if (j == 2) {
475 dest = n->in(i);
476 break;
477 }
478 }
479 }
480 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
481 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
482 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
483 }
484 } else if (strlen(call->_name) > 5 &&
485 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
486 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
487 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
488 }
489 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
490 // skip
571 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
572 Op_VectorizedHashCode,
573 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
574 Op_EncodeISOArray,
575 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
576 Op_CountPositives,
577 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
578 Op_CastP2X,
579 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
580 Op_StrIndexOfChar,
581 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
582 };
583
584 const int others_len = sizeof(others) / sizeof(others[0]);
585 int i = 0;
586 for (; i < others_len; i++) {
587 if (others[i].opcode == n->Opcode()) {
588 break;
589 }
590 }
591 uint stop = n->is_Call() ? n->as_Call()->tf()->domain_sig()->cnt() : n->req();
592 if (i != others_len) {
593 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
594 for (uint j = 0; j < inputs_len; j++) {
595 int pos = others[i].inputs[j].pos;
596 if (pos == -1) {
597 break;
598 }
599 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
600 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
601 }
602 }
603 for (uint j = 1; j < stop; j++) {
604 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
605 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
606 uint k = 0;
607 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
608 if (k == inputs_len) {
609 fatal("arg %d for node %s not covered", j, n->Name());
610 }
611 }
791 mem_ctrl = phase->ctrl_or_self(mem);
792 }
793 return mem;
794 }
795
796 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
797 Node* mem = nullptr;
798 Node* c = ctrl;
799 do {
800 if (c->is_Region()) {
801 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
802 Node* u = c->fast_out(i);
803 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
804 if (u->adr_type() == TypePtr::BOTTOM) {
805 mem = u;
806 }
807 }
808 }
809 } else {
810 if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
811 CallProjections* projs = c->as_Call()->extract_projections(true, false);
812 if (projs->fallthrough_memproj != nullptr) {
813 if (projs->fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
814 if (projs->catchall_memproj == nullptr) {
815 mem = projs->fallthrough_memproj;
816 } else {
817 if (phase->is_dominator(projs->fallthrough_catchproj, ctrl)) {
818 mem = projs->fallthrough_memproj;
819 } else {
820 assert(phase->is_dominator(projs->catchall_catchproj, ctrl), "one proj must dominate barrier");
821 mem = projs->catchall_memproj;
822 }
823 }
824 }
825 } else {
826 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
827 if (proj != nullptr &&
828 proj->adr_type() == TypePtr::BOTTOM) {
829 mem = proj;
830 }
831 }
832 } else {
833 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
834 Node* u = c->fast_out(i);
835 if (u->is_Proj() &&
836 u->bottom_type() == Type::MEMORY &&
837 u->adr_type() == TypePtr::BOTTOM) {
838 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
839 assert(mem == nullptr, "only one proj");
840 mem = u;
841 }
1061 if (c != ctrl ||
1062 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1063 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1064 phase->igvn().rehash_node_delayed(u);
1065 int nb = u->replace_edge(ctrl, region, &phase->igvn());
1066 if (u->is_CFG()) {
1067 if (phase->idom(u) == ctrl) {
1068 phase->set_idom(u, region, phase->dom_depth(region));
1069 }
1070 } else if (phase->get_ctrl(u) == ctrl) {
1071 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1072 uses.push(u);
1073 }
1074 assert(nb == 1, "more than 1 ctrl input?");
1075 --i, imax -= nb;
1076 }
1077 }
1078 }
1079 }
1080
1081 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections* projs, PhaseIdealLoop* phase) {
1082 Node* region = nullptr;
1083 while (c != ctrl) {
1084 if (c->is_Region()) {
1085 region = c;
1086 }
1087 c = phase->idom(c);
1088 }
1089 assert(region != nullptr, "");
1090 Node* phi = new PhiNode(region, n->bottom_type());
1091 for (uint j = 1; j < region->req(); j++) {
1092 Node* in = region->in(j);
1093 if (phase->is_dominator(projs->fallthrough_catchproj, in)) {
1094 phi->init_req(j, n);
1095 } else if (phase->is_dominator(projs->catchall_catchproj, in)) {
1096 phi->init_req(j, n_clone);
1097 } else {
1098 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1099 }
1100 }
1101 phase->register_new_node(phi, region);
1102 return phi;
1103 }
1104
1105 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1106 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1107
1108 Unique_Node_List uses;
1109 for (int i = 0; i < state->iu_barriers_count(); i++) {
1110 Node* barrier = state->iu_barrier(i);
1111 Node* ctrl = phase->get_ctrl(barrier);
1112 IdealLoopTree* loop = phase->get_loop(ctrl);
1113 Node* head = loop->head();
1114 if (head->is_OuterStripMinedLoop()) {
1115 // Expanding a barrier here will break loop strip mining
1191 if (phase->has_ctrl(in)) {
1192 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1193 #ifdef ASSERT
1194 for (uint i = 0; i < stack.size(); i++) {
1195 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1196 }
1197 #endif
1198 stack.push(in, 0);
1199 }
1200 } else {
1201 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1202 }
1203 }
1204 } else {
1205 phase->set_ctrl(n, call->in(0));
1206 stack.pop();
1207 }
1208 } while(stack.size() > 0);
1209 continue;
1210 }
1211 CallProjections* projs = call->extract_projections(false, false);
1212 #ifdef ASSERT
1213 VectorSet cloned;
1214 #endif
1215 Node* lrb_clone = lrb->clone();
1216 phase->register_new_node(lrb_clone, projs->catchall_catchproj);
1217 phase->set_ctrl(lrb, projs->fallthrough_catchproj);
1218
1219 stack.push(lrb, 0);
1220 clones.push(lrb_clone);
1221
1222 do {
1223 assert(stack.size() == clones.size(), "");
1224 Node* n = stack.node();
1225 #ifdef ASSERT
1226 if (n->is_Load()) {
1227 Node* mem = n->in(MemNode::Memory);
1228 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1229 Node* u = mem->fast_out(j);
1230 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1231 }
1232 }
1233 #endif
1234 uint idx = stack.index();
1235 Node* n_clone = clones.at(clones.size()-1);
1236 if (idx < n->outcnt()) {
1237 Node* u = n->raw_out(idx);
1238 Node* c = phase->ctrl_or_self(u);
1239 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs->fallthrough_proj)) {
1240 stack.set_index(idx+1);
1241 assert(!u->is_CFG(), "");
1242 stack.push(u, 0);
1243 assert(!cloned.test_set(u->_idx), "only one clone");
1244 Node* u_clone = u->clone();
1245 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1246 assert(nb > 0, "should have replaced some uses");
1247 phase->register_new_node(u_clone, projs->catchall_catchproj);
1248 clones.push(u_clone);
1249 phase->set_ctrl(u, projs->fallthrough_catchproj);
1250 } else {
1251 bool replaced = false;
1252 if (u->is_Phi()) {
1253 for (uint k = 1; k < u->req(); k++) {
1254 if (u->in(k) == n) {
1255 if (phase->is_dominator(projs->catchall_catchproj, u->in(0)->in(k))) {
1256 phase->igvn().replace_input_of(u, k, n_clone);
1257 replaced = true;
1258 } else if (!phase->is_dominator(projs->fallthrough_catchproj, u->in(0)->in(k))) {
1259 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1260 replaced = true;
1261 }
1262 }
1263 }
1264 } else {
1265 if (phase->is_dominator(projs->catchall_catchproj, c)) {
1266 phase->igvn().rehash_node_delayed(u);
1267 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1268 assert(nb > 0, "should have replaced some uses");
1269 replaced = true;
1270 } else if (!phase->is_dominator(projs->fallthrough_catchproj, c)) {
1271 if (u->is_If()) {
1272 // Can't break If/Bool/Cmp chain
1273 assert(n->is_Bool(), "unexpected If shape");
1274 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1275 assert(n_clone->is_Bool(), "unexpected clone");
1276 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1277 Node* bol_clone = n->clone();
1278 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1279 bol_clone->set_req(1, cmp_clone);
1280
1281 Node* nn = stack.node_at(stack.size()-3);
1282 Node* nn_clone = clones.at(clones.size()-3);
1283 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1284
1285 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1286 &phase->igvn());
1287 assert(nb > 0, "should have replaced some uses");
1288
1289 phase->register_new_node(bol_clone, u->in(0));
1290 phase->register_new_node(cmp_clone, u->in(0));
2321 } else {
2322 assert(c != c->in(0), "");
2323 c = c->in(0);
2324 }
2325 }
2326 }
2327 }
2328 } while (stack.size() > 0);
2329 assert(mem != nullptr, "should have found safepoint");
2330 } else {
2331 mem = phi_mem;
2332 }
2333 return mem;
2334 }
2335
2336 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
2337 Node* c = _phase->get_ctrl(n);
2338 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
2339 assert(c == n->in(0), "");
2340 CallNode* call = c->as_Call();
2341 CallProjections* projs = call->extract_projections(true, false);
2342 if (projs->catchall_memproj != nullptr) {
2343 if (projs->fallthrough_memproj == n) {
2344 c = projs->fallthrough_catchproj;
2345 } else {
2346 assert(projs->catchall_memproj == n, "");
2347 c = projs->catchall_catchproj;
2348 }
2349 }
2350 }
2351 return c;
2352 }
2353
2354 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
2355 if (_phase->has_ctrl(n))
2356 return get_ctrl(n);
2357 else {
2358 assert (n->is_CFG(), "must be a CFG node");
2359 return n;
2360 }
2361 }
2362
2363 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2364 return m != nullptr && get_ctrl(m) == c;
2365 }
2366
2367 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|