446 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
447 "sha1_implCompressMB",
448 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
449 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
450 "sha256_implCompressMB",
451 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
452 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
453 "sha512_implCompressMB",
454 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
455 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
456 "encodeBlock",
457 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
458 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
459 "decodeBlock",
460 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
461 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
462 };
463
464 if (call->is_call_to_arraycopystub()) {
465 Node* dest = NULL;
466 const TypeTuple* args = n->as_Call()->_tf->domain();
467 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
468 if (args->field_at(i)->isa_ptr()) {
469 j++;
470 if (j == 2) {
471 dest = n->in(i);
472 break;
473 }
474 }
475 }
476 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
477 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
478 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
479 }
480 } else if (strlen(call->_name) > 5 &&
481 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
482 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
483 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
484 }
485 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
486 // skip
565 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } },
566 Op_StrEquals,
567 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
568 Op_EncodeISOArray,
569 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
570 Op_CountPositives,
571 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
572 Op_CastP2X,
573 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
574 Op_StrIndexOfChar,
575 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
576 };
577
578 const int others_len = sizeof(others) / sizeof(others[0]);
579 int i = 0;
580 for (; i < others_len; i++) {
581 if (others[i].opcode == n->Opcode()) {
582 break;
583 }
584 }
585 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
586 if (i != others_len) {
587 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
588 for (uint j = 0; j < inputs_len; j++) {
589 int pos = others[i].inputs[j].pos;
590 if (pos == -1) {
591 break;
592 }
593 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
594 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
595 }
596 }
597 for (uint j = 1; j < stop; j++) {
598 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
599 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
600 uint k = 0;
601 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
602 if (k == inputs_len) {
603 fatal("arg %d for node %s not covered", j, n->Name());
604 }
605 }
785 mem_ctrl = phase->ctrl_or_self(mem);
786 }
787 return mem;
788 }
789
790 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
791 Node* mem = NULL;
792 Node* c = ctrl;
793 do {
794 if (c->is_Region()) {
795 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) {
796 Node* u = c->fast_out(i);
797 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
798 if (u->adr_type() == TypePtr::BOTTOM) {
799 mem = u;
800 }
801 }
802 }
803 } else {
804 if (c->is_Call() && c->as_Call()->adr_type() != NULL) {
805 CallProjections projs;
806 c->as_Call()->extract_projections(&projs, true, false);
807 if (projs.fallthrough_memproj != NULL) {
808 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
809 if (projs.catchall_memproj == NULL) {
810 mem = projs.fallthrough_memproj;
811 } else {
812 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
813 mem = projs.fallthrough_memproj;
814 } else {
815 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
816 mem = projs.catchall_memproj;
817 }
818 }
819 }
820 } else {
821 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
822 if (proj != NULL &&
823 proj->adr_type() == TypePtr::BOTTOM) {
824 mem = proj;
825 }
826 }
827 } else {
828 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
829 Node* u = c->fast_out(i);
830 if (u->is_Proj() &&
831 u->bottom_type() == Type::MEMORY &&
832 u->adr_type() == TypePtr::BOTTOM) {
833 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
834 assert(mem == NULL, "only one proj");
835 mem = u;
836 }
1056 if (c != ctrl ||
1057 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1058 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1059 phase->igvn().rehash_node_delayed(u);
1060 int nb = u->replace_edge(ctrl, region, &phase->igvn());
1061 if (u->is_CFG()) {
1062 if (phase->idom(u) == ctrl) {
1063 phase->set_idom(u, region, phase->dom_depth(region));
1064 }
1065 } else if (phase->get_ctrl(u) == ctrl) {
1066 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1067 uses.push(u);
1068 }
1069 assert(nb == 1, "more than 1 ctrl input?");
1070 --i, imax -= nb;
1071 }
1072 }
1073 }
1074 }
1075
1076 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
1077 Node* region = NULL;
1078 while (c != ctrl) {
1079 if (c->is_Region()) {
1080 region = c;
1081 }
1082 c = phase->idom(c);
1083 }
1084 assert(region != NULL, "");
1085 Node* phi = new PhiNode(region, n->bottom_type());
1086 for (uint j = 1; j < region->req(); j++) {
1087 Node* in = region->in(j);
1088 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1089 phi->init_req(j, n);
1090 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1091 phi->init_req(j, n_clone);
1092 } else {
1093 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1094 }
1095 }
1096 phase->register_new_node(phi, region);
1097 return phi;
1098 }
1099
1100 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1101 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1102
1103 Unique_Node_List uses;
1104 for (int i = 0; i < state->iu_barriers_count(); i++) {
1105 Node* barrier = state->iu_barrier(i);
1106 Node* ctrl = phase->get_ctrl(barrier);
1107 IdealLoopTree* loop = phase->get_loop(ctrl);
1108 Node* head = loop->head();
1109 if (head->is_OuterStripMinedLoop()) {
1110 // Expanding a barrier here will break loop strip mining
1186 if (phase->has_ctrl(in)) {
1187 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1188 #ifdef ASSERT
1189 for (uint i = 0; i < stack.size(); i++) {
1190 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1191 }
1192 #endif
1193 stack.push(in, 0);
1194 }
1195 } else {
1196 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1197 }
1198 }
1199 } else {
1200 phase->set_ctrl(n, call->in(0));
1201 stack.pop();
1202 }
1203 } while(stack.size() > 0);
1204 continue;
1205 }
1206 CallProjections projs;
1207 call->extract_projections(&projs, false, false);
1208
1209 #ifdef ASSERT
1210 VectorSet cloned;
1211 #endif
1212 Node* lrb_clone = lrb->clone();
1213 phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1214 phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1215
1216 stack.push(lrb, 0);
1217 clones.push(lrb_clone);
1218
1219 do {
1220 assert(stack.size() == clones.size(), "");
1221 Node* n = stack.node();
1222 #ifdef ASSERT
1223 if (n->is_Load()) {
1224 Node* mem = n->in(MemNode::Memory);
1225 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1226 Node* u = mem->fast_out(j);
1227 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1228 }
1229 }
1230 #endif
1231 uint idx = stack.index();
1232 Node* n_clone = clones.at(clones.size()-1);
1233 if (idx < n->outcnt()) {
1234 Node* u = n->raw_out(idx);
1235 Node* c = phase->ctrl_or_self(u);
1236 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) {
1237 stack.set_index(idx+1);
1238 assert(!u->is_CFG(), "");
1239 stack.push(u, 0);
1240 assert(!cloned.test_set(u->_idx), "only one clone");
1241 Node* u_clone = u->clone();
1242 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1243 assert(nb > 0, "should have replaced some uses");
1244 phase->register_new_node(u_clone, projs.catchall_catchproj);
1245 clones.push(u_clone);
1246 phase->set_ctrl(u, projs.fallthrough_catchproj);
1247 } else {
1248 bool replaced = false;
1249 if (u->is_Phi()) {
1250 for (uint k = 1; k < u->req(); k++) {
1251 if (u->in(k) == n) {
1252 if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1253 phase->igvn().replace_input_of(u, k, n_clone);
1254 replaced = true;
1255 } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1256 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1257 replaced = true;
1258 }
1259 }
1260 }
1261 } else {
1262 if (phase->is_dominator(projs.catchall_catchproj, c)) {
1263 phase->igvn().rehash_node_delayed(u);
1264 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1265 assert(nb > 0, "should have replaced some uses");
1266 replaced = true;
1267 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1268 if (u->is_If()) {
1269 // Can't break If/Bool/Cmp chain
1270 assert(n->is_Bool(), "unexpected If shape");
1271 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1272 assert(n_clone->is_Bool(), "unexpected clone");
1273 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1274 Node* bol_clone = n->clone();
1275 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1276 bol_clone->set_req(1, cmp_clone);
1277
1278 Node* nn = stack.node_at(stack.size()-3);
1279 Node* nn_clone = clones.at(clones.size()-3);
1280 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1281
1282 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1283 &phase->igvn());
1284 assert(nb > 0, "should have replaced some uses");
1285
1286 phase->register_new_node(bol_clone, u->in(0));
1287 phase->register_new_node(cmp_clone, u->in(0));
2320 } else {
2321 assert(c != c->in(0), "");
2322 c = c->in(0);
2323 }
2324 }
2325 }
2326 }
2327 } while (stack.size() > 0);
2328 assert(mem != NULL, "should have found safepoint");
2329 } else {
2330 mem = phi_mem;
2331 }
2332 return mem;
2333 }
2334
2335 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
2336 Node* c = _phase->get_ctrl(n);
2337 if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Call()) {
2338 assert(c == n->in(0), "");
2339 CallNode* call = c->as_Call();
2340 CallProjections projs;
2341 call->extract_projections(&projs, true, false);
2342 if (projs.catchall_memproj != NULL) {
2343 if (projs.fallthrough_memproj == n) {
2344 c = projs.fallthrough_catchproj;
2345 } else {
2346 assert(projs.catchall_memproj == n, "");
2347 c = projs.catchall_catchproj;
2348 }
2349 }
2350 }
2351 return c;
2352 }
2353
2354 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
2355 if (_phase->has_ctrl(n))
2356 return get_ctrl(n);
2357 else {
2358 assert (n->is_CFG(), "must be a CFG node");
2359 return n;
2360 }
2361 }
2362
2363 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2364 return m != NULL && get_ctrl(m) == c;
2365 }
2366
2367 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|
446 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
447 "sha1_implCompressMB",
448 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
449 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
450 "sha256_implCompressMB",
451 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
452 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
453 "sha512_implCompressMB",
454 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
455 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
456 "encodeBlock",
457 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
458 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
459 "decodeBlock",
460 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
461 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
462 };
463
464 if (call->is_call_to_arraycopystub()) {
465 Node* dest = NULL;
466 const TypeTuple* args = n->as_Call()->_tf->domain_sig();
467 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
468 if (args->field_at(i)->isa_ptr()) {
469 j++;
470 if (j == 2) {
471 dest = n->in(i);
472 break;
473 }
474 }
475 }
476 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
477 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
478 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
479 }
480 } else if (strlen(call->_name) > 5 &&
481 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
482 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
483 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
484 }
485 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
486 // skip
565 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } },
566 Op_StrEquals,
567 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
568 Op_EncodeISOArray,
569 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
570 Op_CountPositives,
571 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
572 Op_CastP2X,
573 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
574 Op_StrIndexOfChar,
575 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
576 };
577
578 const int others_len = sizeof(others) / sizeof(others[0]);
579 int i = 0;
580 for (; i < others_len; i++) {
581 if (others[i].opcode == n->Opcode()) {
582 break;
583 }
584 }
585 uint stop = n->is_Call() ? n->as_Call()->tf()->domain_sig()->cnt() : n->req();
586 if (i != others_len) {
587 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
588 for (uint j = 0; j < inputs_len; j++) {
589 int pos = others[i].inputs[j].pos;
590 if (pos == -1) {
591 break;
592 }
593 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
594 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
595 }
596 }
597 for (uint j = 1; j < stop; j++) {
598 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
599 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
600 uint k = 0;
601 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
602 if (k == inputs_len) {
603 fatal("arg %d for node %s not covered", j, n->Name());
604 }
605 }
785 mem_ctrl = phase->ctrl_or_self(mem);
786 }
787 return mem;
788 }
789
790 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
791 Node* mem = NULL;
792 Node* c = ctrl;
793 do {
794 if (c->is_Region()) {
795 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) {
796 Node* u = c->fast_out(i);
797 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
798 if (u->adr_type() == TypePtr::BOTTOM) {
799 mem = u;
800 }
801 }
802 }
803 } else {
804 if (c->is_Call() && c->as_Call()->adr_type() != NULL) {
805 CallProjections* projs = c->as_Call()->extract_projections(true, false);
806 if (projs->fallthrough_memproj != NULL) {
807 if (projs->fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
808 if (projs->catchall_memproj == NULL) {
809 mem = projs->fallthrough_memproj;
810 } else {
811 if (phase->is_dominator(projs->fallthrough_catchproj, ctrl)) {
812 mem = projs->fallthrough_memproj;
813 } else {
814 assert(phase->is_dominator(projs->catchall_catchproj, ctrl), "one proj must dominate barrier");
815 mem = projs->catchall_memproj;
816 }
817 }
818 }
819 } else {
820 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
821 if (proj != NULL &&
822 proj->adr_type() == TypePtr::BOTTOM) {
823 mem = proj;
824 }
825 }
826 } else {
827 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
828 Node* u = c->fast_out(i);
829 if (u->is_Proj() &&
830 u->bottom_type() == Type::MEMORY &&
831 u->adr_type() == TypePtr::BOTTOM) {
832 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
833 assert(mem == NULL, "only one proj");
834 mem = u;
835 }
1055 if (c != ctrl ||
1056 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1057 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1058 phase->igvn().rehash_node_delayed(u);
1059 int nb = u->replace_edge(ctrl, region, &phase->igvn());
1060 if (u->is_CFG()) {
1061 if (phase->idom(u) == ctrl) {
1062 phase->set_idom(u, region, phase->dom_depth(region));
1063 }
1064 } else if (phase->get_ctrl(u) == ctrl) {
1065 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1066 uses.push(u);
1067 }
1068 assert(nb == 1, "more than 1 ctrl input?");
1069 --i, imax -= nb;
1070 }
1071 }
1072 }
1073 }
1074
1075 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections* projs, PhaseIdealLoop* phase) {
1076 Node* region = NULL;
1077 while (c != ctrl) {
1078 if (c->is_Region()) {
1079 region = c;
1080 }
1081 c = phase->idom(c);
1082 }
1083 assert(region != NULL, "");
1084 Node* phi = new PhiNode(region, n->bottom_type());
1085 for (uint j = 1; j < region->req(); j++) {
1086 Node* in = region->in(j);
1087 if (phase->is_dominator(projs->fallthrough_catchproj, in)) {
1088 phi->init_req(j, n);
1089 } else if (phase->is_dominator(projs->catchall_catchproj, in)) {
1090 phi->init_req(j, n_clone);
1091 } else {
1092 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1093 }
1094 }
1095 phase->register_new_node(phi, region);
1096 return phi;
1097 }
1098
1099 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1100 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1101
1102 Unique_Node_List uses;
1103 for (int i = 0; i < state->iu_barriers_count(); i++) {
1104 Node* barrier = state->iu_barrier(i);
1105 Node* ctrl = phase->get_ctrl(barrier);
1106 IdealLoopTree* loop = phase->get_loop(ctrl);
1107 Node* head = loop->head();
1108 if (head->is_OuterStripMinedLoop()) {
1109 // Expanding a barrier here will break loop strip mining
1185 if (phase->has_ctrl(in)) {
1186 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1187 #ifdef ASSERT
1188 for (uint i = 0; i < stack.size(); i++) {
1189 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1190 }
1191 #endif
1192 stack.push(in, 0);
1193 }
1194 } else {
1195 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1196 }
1197 }
1198 } else {
1199 phase->set_ctrl(n, call->in(0));
1200 stack.pop();
1201 }
1202 } while(stack.size() > 0);
1203 continue;
1204 }
1205 CallProjections* projs = call->extract_projections(false, false);
1206 #ifdef ASSERT
1207 VectorSet cloned;
1208 #endif
1209 Node* lrb_clone = lrb->clone();
1210 phase->register_new_node(lrb_clone, projs->catchall_catchproj);
1211 phase->set_ctrl(lrb, projs->fallthrough_catchproj);
1212
1213 stack.push(lrb, 0);
1214 clones.push(lrb_clone);
1215
1216 do {
1217 assert(stack.size() == clones.size(), "");
1218 Node* n = stack.node();
1219 #ifdef ASSERT
1220 if (n->is_Load()) {
1221 Node* mem = n->in(MemNode::Memory);
1222 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1223 Node* u = mem->fast_out(j);
1224 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1225 }
1226 }
1227 #endif
1228 uint idx = stack.index();
1229 Node* n_clone = clones.at(clones.size()-1);
1230 if (idx < n->outcnt()) {
1231 Node* u = n->raw_out(idx);
1232 Node* c = phase->ctrl_or_self(u);
1233 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs->fallthrough_proj)) {
1234 stack.set_index(idx+1);
1235 assert(!u->is_CFG(), "");
1236 stack.push(u, 0);
1237 assert(!cloned.test_set(u->_idx), "only one clone");
1238 Node* u_clone = u->clone();
1239 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1240 assert(nb > 0, "should have replaced some uses");
1241 phase->register_new_node(u_clone, projs->catchall_catchproj);
1242 clones.push(u_clone);
1243 phase->set_ctrl(u, projs->fallthrough_catchproj);
1244 } else {
1245 bool replaced = false;
1246 if (u->is_Phi()) {
1247 for (uint k = 1; k < u->req(); k++) {
1248 if (u->in(k) == n) {
1249 if (phase->is_dominator(projs->catchall_catchproj, u->in(0)->in(k))) {
1250 phase->igvn().replace_input_of(u, k, n_clone);
1251 replaced = true;
1252 } else if (!phase->is_dominator(projs->fallthrough_catchproj, u->in(0)->in(k))) {
1253 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1254 replaced = true;
1255 }
1256 }
1257 }
1258 } else {
1259 if (phase->is_dominator(projs->catchall_catchproj, c)) {
1260 phase->igvn().rehash_node_delayed(u);
1261 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1262 assert(nb > 0, "should have replaced some uses");
1263 replaced = true;
1264 } else if (!phase->is_dominator(projs->fallthrough_catchproj, c)) {
1265 if (u->is_If()) {
1266 // Can't break If/Bool/Cmp chain
1267 assert(n->is_Bool(), "unexpected If shape");
1268 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1269 assert(n_clone->is_Bool(), "unexpected clone");
1270 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1271 Node* bol_clone = n->clone();
1272 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1273 bol_clone->set_req(1, cmp_clone);
1274
1275 Node* nn = stack.node_at(stack.size()-3);
1276 Node* nn_clone = clones.at(clones.size()-3);
1277 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1278
1279 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1280 &phase->igvn());
1281 assert(nb > 0, "should have replaced some uses");
1282
1283 phase->register_new_node(bol_clone, u->in(0));
1284 phase->register_new_node(cmp_clone, u->in(0));
2317 } else {
2318 assert(c != c->in(0), "");
2319 c = c->in(0);
2320 }
2321 }
2322 }
2323 }
2324 } while (stack.size() > 0);
2325 assert(mem != NULL, "should have found safepoint");
2326 } else {
2327 mem = phi_mem;
2328 }
2329 return mem;
2330 }
2331
2332 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
2333 Node* c = _phase->get_ctrl(n);
2334 if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Call()) {
2335 assert(c == n->in(0), "");
2336 CallNode* call = c->as_Call();
2337 CallProjections* projs = call->extract_projections(true, false);
2338 if (projs->catchall_memproj != NULL) {
2339 if (projs->fallthrough_memproj == n) {
2340 c = projs->fallthrough_catchproj;
2341 } else {
2342 assert(projs->catchall_memproj == n, "");
2343 c = projs->catchall_catchproj;
2344 }
2345 }
2346 }
2347 return c;
2348 }
2349
2350 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
2351 if (_phase->has_ctrl(n))
2352 return get_ctrl(n);
2353 else {
2354 assert (n->is_CFG(), "must be a CFG node");
2355 return n;
2356 }
2357 }
2358
2359 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2360 return m != NULL && get_ctrl(m) == c;
2361 }
2362
2363 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|