431 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
432 "sha512_implCompressMB",
433 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
434 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
435 "encodeBlock",
436 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
437 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
438 "decodeBlock",
439 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
440 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
441 "intpoly_montgomeryMult_P256",
442 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahStore },
443 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
444 "intpoly_assign",
445 { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone },
446 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
447 };
448
449 if (call->is_call_to_arraycopystub()) {
450 Node* dest = nullptr;
451 const TypeTuple* args = n->as_Call()->_tf->domain();
452 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
453 if (args->field_at(i)->isa_ptr()) {
454 j++;
455 if (j == 2) {
456 dest = n->in(i);
457 break;
458 }
459 }
460 }
461 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
462 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
463 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
464 }
465 } else if (strlen(call->_name) > 5 &&
466 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
467 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
468 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
469 }
470 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
471 // skip
552 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
553 Op_VectorizedHashCode,
554 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
555 Op_EncodeISOArray,
556 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
557 Op_CountPositives,
558 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
559 Op_CastP2X,
560 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
561 Op_StrIndexOfChar,
562 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
563 };
564
565 const int others_len = sizeof(others) / sizeof(others[0]);
566 int i = 0;
567 for (; i < others_len; i++) {
568 if (others[i].opcode == n->Opcode()) {
569 break;
570 }
571 }
572 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
573 if (i != others_len) {
574 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
575 for (uint j = 0; j < inputs_len; j++) {
576 int pos = others[i].inputs[j].pos;
577 if (pos == -1) {
578 break;
579 }
580 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
581 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
582 }
583 }
584 for (uint j = 1; j < stop; j++) {
585 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
586 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
587 uint k = 0;
588 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
589 if (k == inputs_len) {
590 fatal("arg %d for node %s not covered", j, n->Name());
591 }
592 }
787 mem_ctrl = phase->ctrl_or_self(mem);
788 }
789 return mem;
790 }
791
792 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
793 Node* mem = nullptr;
794 Node* c = ctrl;
795 do {
796 if (c->is_Region()) {
797 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
798 Node* u = c->fast_out(i);
799 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
800 if (u->adr_type() == TypePtr::BOTTOM) {
801 mem = u;
802 }
803 }
804 }
805 } else {
806 if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
807 CallProjections projs;
808 c->as_Call()->extract_projections(&projs, true, false);
809 if (projs.fallthrough_memproj != nullptr) {
810 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
811 if (projs.catchall_memproj == nullptr) {
812 mem = projs.fallthrough_memproj;
813 } else {
814 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
815 mem = projs.fallthrough_memproj;
816 } else {
817 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
818 mem = projs.catchall_memproj;
819 }
820 }
821 }
822 } else {
823 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
824 if (proj != nullptr &&
825 proj->adr_type() == TypePtr::BOTTOM) {
826 mem = proj;
827 }
828 }
829 } else {
830 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
831 Node* u = c->fast_out(i);
832 if (u->is_Proj() &&
833 u->bottom_type() == Type::MEMORY &&
834 u->adr_type() == TypePtr::BOTTOM) {
835 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
836 assert(mem == nullptr, "only one proj");
837 mem = u;
838 }
1078 if (old_c != ctrl ||
1079 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1080 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1081 phase->igvn().rehash_node_delayed(u);
1082 int nb = u->replace_edge(ctrl, region, &phase->igvn());
1083 if (u->is_CFG()) {
1084 if (phase->idom(u) == ctrl) {
1085 phase->set_idom(u, region, phase->dom_depth(region));
1086 }
1087 } else if (phase->get_ctrl(u) == ctrl) {
1088 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1089 uses.push(u);
1090 }
1091 assert(nb == 1, "more than 1 ctrl input?");
1092 --i, imax -= nb;
1093 }
1094 }
1095 }
1096 }
1097
1098 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
1099 Node* region = nullptr;
1100 while (c != ctrl) {
1101 if (c->is_Region()) {
1102 region = c;
1103 }
1104 c = phase->idom(c);
1105 }
1106 assert(region != nullptr, "");
1107 Node* phi = new PhiNode(region, n->bottom_type());
1108 for (uint j = 1; j < region->req(); j++) {
1109 Node* in = region->in(j);
1110 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1111 phi->init_req(j, n);
1112 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1113 phi->init_req(j, n_clone);
1114 } else {
1115 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1116 }
1117 }
1118 phase->register_new_node(phi, region);
1119 return phi;
1120 }
1121
1122 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1123 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1124
1125 Unique_Node_List uses;
1126 Node_Stack stack(0);
1127 Node_List clones;
1128 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1129 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1130
1131 Node* ctrl = phase->get_ctrl(lrb);
1132 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1195 if (phase->has_ctrl(in)) {
1196 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1197 #ifdef ASSERT
1198 for (uint i = 0; i < stack.size(); i++) {
1199 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1200 }
1201 #endif
1202 stack.push(in, 0);
1203 }
1204 } else {
1205 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1206 }
1207 }
1208 } else {
1209 phase->set_ctrl(n, call->in(0));
1210 stack.pop();
1211 }
1212 } while(stack.size() > 0);
1213 continue;
1214 }
1215 CallProjections projs;
1216 call->extract_projections(&projs, false, false);
1217
1218 // If this is a runtime call, it doesn't have an exception handling path
1219 if (projs.fallthrough_catchproj == nullptr) {
1220 assert(call->method() == nullptr, "should be runtime call");
1221 assert(projs.catchall_catchproj == nullptr, "runtime call should not have catch all projection");
1222 continue;
1223 }
1224
1225 // Otherwise, clone the barrier so there's one for the fallthrough and one for the exception handling path
1226 #ifdef ASSERT
1227 VectorSet cloned;
1228 #endif
1229 Node* lrb_clone = lrb->clone();
1230 phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1231 phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1232
1233 stack.push(lrb, 0);
1234 clones.push(lrb_clone);
1235
1236 do {
1237 assert(stack.size() == clones.size(), "");
1238 Node* n = stack.node();
1239 #ifdef ASSERT
1240 if (n->is_Load()) {
1241 Node* mem = n->in(MemNode::Memory);
1242 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1243 Node* u = mem->fast_out(j);
1244 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1245 }
1246 }
1247 #endif
1248 uint idx = stack.index();
1249 Node* n_clone = clones.at(clones.size()-1);
1250 if (idx < n->outcnt()) {
1251 Node* u = n->raw_out(idx);
1252 Node* c = phase->ctrl_or_self(u);
1253 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) {
1254 stack.set_index(idx+1);
1255 assert(!u->is_CFG(), "");
1256 stack.push(u, 0);
1257 assert(!cloned.test_set(u->_idx), "only one clone");
1258 Node* u_clone = u->clone();
1259 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1260 assert(nb > 0, "should have replaced some uses");
1261 phase->register_new_node(u_clone, projs.catchall_catchproj);
1262 clones.push(u_clone);
1263 phase->set_ctrl(u, projs.fallthrough_catchproj);
1264 } else {
1265 bool replaced = false;
1266 if (u->is_Phi()) {
1267 for (uint k = 1; k < u->req(); k++) {
1268 if (u->in(k) == n) {
1269 if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1270 phase->igvn().replace_input_of(u, k, n_clone);
1271 replaced = true;
1272 } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1273 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1274 replaced = true;
1275 }
1276 }
1277 }
1278 } else {
1279 if (phase->is_dominator(projs.catchall_catchproj, c)) {
1280 phase->igvn().rehash_node_delayed(u);
1281 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1282 assert(nb > 0, "should have replaced some uses");
1283 replaced = true;
1284 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1285 if (u->is_If()) {
1286 // Can't break If/Bool/Cmp chain
1287 assert(n->is_Bool(), "unexpected If shape");
1288 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1289 assert(n_clone->is_Bool(), "unexpected clone");
1290 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1291 Node* bol_clone = n->clone();
1292 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1293 bol_clone->set_req(1, cmp_clone);
1294
1295 Node* nn = stack.node_at(stack.size()-3);
1296 Node* nn_clone = clones.at(clones.size()-3);
1297 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1298
1299 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1300 &phase->igvn());
1301 assert(nb > 0, "should have replaced some uses");
1302
1303 phase->register_new_node(bol_clone, u->in(0));
1304 phase->register_new_node(cmp_clone, u->in(0));
1855 } else {
1856 assert(c != c->in(0), "");
1857 c = c->in(0);
1858 }
1859 }
1860 }
1861 }
1862 } while (stack.size() > 0);
1863 assert(mem != nullptr, "should have found safepoint");
1864 } else {
1865 mem = phi_mem;
1866 }
1867 return mem;
1868 }
1869
1870 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
1871 Node* c = _phase->get_ctrl(n);
1872 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
1873 assert(c == n->in(0), "");
1874 CallNode* call = c->as_Call();
1875 CallProjections projs;
1876 call->extract_projections(&projs, true, false);
1877 if (projs.catchall_memproj != nullptr) {
1878 if (projs.fallthrough_memproj == n) {
1879 c = projs.fallthrough_catchproj;
1880 } else {
1881 assert(projs.catchall_memproj == n, "");
1882 c = projs.catchall_catchproj;
1883 }
1884 }
1885 }
1886 return c;
1887 }
1888
1889 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
1890 if (_phase->has_ctrl(n))
1891 return get_ctrl(n);
1892 else {
1893 assert (n->is_CFG(), "must be a CFG node");
1894 return n;
1895 }
1896 }
1897
1898 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
1899 return m != nullptr && get_ctrl(m) == c;
1900 }
1901
1902 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|
431 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
432 "sha512_implCompressMB",
433 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
434 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
435 "encodeBlock",
436 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
437 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
438 "decodeBlock",
439 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
440 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
441 "intpoly_montgomeryMult_P256",
442 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahStore },
443 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
444 "intpoly_assign",
445 { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone },
446 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
447 };
448
449 if (call->is_call_to_arraycopystub()) {
450 Node* dest = nullptr;
451 const TypeTuple* args = n->as_Call()->_tf->domain_sig();
452 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
453 if (args->field_at(i)->isa_ptr()) {
454 j++;
455 if (j == 2) {
456 dest = n->in(i);
457 break;
458 }
459 }
460 }
461 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
462 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
463 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
464 }
465 } else if (strlen(call->_name) > 5 &&
466 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
467 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
468 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
469 }
470 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
471 // skip
552 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
553 Op_VectorizedHashCode,
554 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
555 Op_EncodeISOArray,
556 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
557 Op_CountPositives,
558 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
559 Op_CastP2X,
560 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
561 Op_StrIndexOfChar,
562 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
563 };
564
565 const int others_len = sizeof(others) / sizeof(others[0]);
566 int i = 0;
567 for (; i < others_len; i++) {
568 if (others[i].opcode == n->Opcode()) {
569 break;
570 }
571 }
572 uint stop = n->is_Call() ? n->as_Call()->tf()->domain_sig()->cnt() : n->req();
573 if (i != others_len) {
574 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
575 for (uint j = 0; j < inputs_len; j++) {
576 int pos = others[i].inputs[j].pos;
577 if (pos == -1) {
578 break;
579 }
580 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
581 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
582 }
583 }
584 for (uint j = 1; j < stop; j++) {
585 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
586 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
587 uint k = 0;
588 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
589 if (k == inputs_len) {
590 fatal("arg %d for node %s not covered", j, n->Name());
591 }
592 }
787 mem_ctrl = phase->ctrl_or_self(mem);
788 }
789 return mem;
790 }
791
792 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
793 Node* mem = nullptr;
794 Node* c = ctrl;
795 do {
796 if (c->is_Region()) {
797 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
798 Node* u = c->fast_out(i);
799 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
800 if (u->adr_type() == TypePtr::BOTTOM) {
801 mem = u;
802 }
803 }
804 }
805 } else {
806 if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
807 CallProjections* projs = c->as_Call()->extract_projections(true, false);
808 if (projs->fallthrough_memproj != nullptr) {
809 if (projs->fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
810 if (projs->catchall_memproj == nullptr) {
811 mem = projs->fallthrough_memproj;
812 } else {
813 if (phase->is_dominator(projs->fallthrough_catchproj, ctrl)) {
814 mem = projs->fallthrough_memproj;
815 } else {
816 assert(phase->is_dominator(projs->catchall_catchproj, ctrl), "one proj must dominate barrier");
817 mem = projs->catchall_memproj;
818 }
819 }
820 }
821 } else {
822 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
823 if (proj != nullptr &&
824 proj->adr_type() == TypePtr::BOTTOM) {
825 mem = proj;
826 }
827 }
828 } else {
829 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
830 Node* u = c->fast_out(i);
831 if (u->is_Proj() &&
832 u->bottom_type() == Type::MEMORY &&
833 u->adr_type() == TypePtr::BOTTOM) {
834 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
835 assert(mem == nullptr, "only one proj");
836 mem = u;
837 }
1077 if (old_c != ctrl ||
1078 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1079 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1080 phase->igvn().rehash_node_delayed(u);
1081 int nb = u->replace_edge(ctrl, region, &phase->igvn());
1082 if (u->is_CFG()) {
1083 if (phase->idom(u) == ctrl) {
1084 phase->set_idom(u, region, phase->dom_depth(region));
1085 }
1086 } else if (phase->get_ctrl(u) == ctrl) {
1087 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1088 uses.push(u);
1089 }
1090 assert(nb == 1, "more than 1 ctrl input?");
1091 --i, imax -= nb;
1092 }
1093 }
1094 }
1095 }
1096
1097 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections* projs, PhaseIdealLoop* phase) {
1098 Node* region = nullptr;
1099 while (c != ctrl) {
1100 if (c->is_Region()) {
1101 region = c;
1102 }
1103 c = phase->idom(c);
1104 }
1105 assert(region != nullptr, "");
1106 Node* phi = new PhiNode(region, n->bottom_type());
1107 for (uint j = 1; j < region->req(); j++) {
1108 Node* in = region->in(j);
1109 if (phase->is_dominator(projs->fallthrough_catchproj, in)) {
1110 phi->init_req(j, n);
1111 } else if (phase->is_dominator(projs->catchall_catchproj, in)) {
1112 phi->init_req(j, n_clone);
1113 } else {
1114 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1115 }
1116 }
1117 phase->register_new_node(phi, region);
1118 return phi;
1119 }
1120
1121 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1122 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1123
1124 Unique_Node_List uses;
1125 Node_Stack stack(0);
1126 Node_List clones;
1127 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1128 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1129
1130 Node* ctrl = phase->get_ctrl(lrb);
1131 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1194 if (phase->has_ctrl(in)) {
1195 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1196 #ifdef ASSERT
1197 for (uint i = 0; i < stack.size(); i++) {
1198 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1199 }
1200 #endif
1201 stack.push(in, 0);
1202 }
1203 } else {
1204 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1205 }
1206 }
1207 } else {
1208 phase->set_ctrl(n, call->in(0));
1209 stack.pop();
1210 }
1211 } while(stack.size() > 0);
1212 continue;
1213 }
1214 CallProjections* projs = call->extract_projections(false, false);
1215
1216 // If this is a runtime call, it doesn't have an exception handling path
1217 if (projs->fallthrough_catchproj == nullptr) {
1218 assert(call->method() == nullptr, "should be runtime call");
1219 assert(projs->catchall_catchproj == nullptr, "runtime call should not have catch all projection");
1220 continue;
1221 }
1222
1223 // Otherwise, clone the barrier so there's one for the fallthrough and one for the exception handling path
1224 #ifdef ASSERT
1225 VectorSet cloned;
1226 #endif
1227 Node* lrb_clone = lrb->clone();
1228 phase->register_new_node(lrb_clone, projs->catchall_catchproj);
1229 phase->set_ctrl(lrb, projs->fallthrough_catchproj);
1230
1231 stack.push(lrb, 0);
1232 clones.push(lrb_clone);
1233
1234 do {
1235 assert(stack.size() == clones.size(), "");
1236 Node* n = stack.node();
1237 #ifdef ASSERT
1238 if (n->is_Load()) {
1239 Node* mem = n->in(MemNode::Memory);
1240 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1241 Node* u = mem->fast_out(j);
1242 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1243 }
1244 }
1245 #endif
1246 uint idx = stack.index();
1247 Node* n_clone = clones.at(clones.size()-1);
1248 if (idx < n->outcnt()) {
1249 Node* u = n->raw_out(idx);
1250 Node* c = phase->ctrl_or_self(u);
1251 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs->fallthrough_proj)) {
1252 stack.set_index(idx+1);
1253 assert(!u->is_CFG(), "");
1254 stack.push(u, 0);
1255 assert(!cloned.test_set(u->_idx), "only one clone");
1256 Node* u_clone = u->clone();
1257 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1258 assert(nb > 0, "should have replaced some uses");
1259 phase->register_new_node(u_clone, projs->catchall_catchproj);
1260 clones.push(u_clone);
1261 phase->set_ctrl(u, projs->fallthrough_catchproj);
1262 } else {
1263 bool replaced = false;
1264 if (u->is_Phi()) {
1265 for (uint k = 1; k < u->req(); k++) {
1266 if (u->in(k) == n) {
1267 if (phase->is_dominator(projs->catchall_catchproj, u->in(0)->in(k))) {
1268 phase->igvn().replace_input_of(u, k, n_clone);
1269 replaced = true;
1270 } else if (!phase->is_dominator(projs->fallthrough_catchproj, u->in(0)->in(k))) {
1271 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1272 replaced = true;
1273 }
1274 }
1275 }
1276 } else {
1277 if (phase->is_dominator(projs->catchall_catchproj, c)) {
1278 phase->igvn().rehash_node_delayed(u);
1279 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1280 assert(nb > 0, "should have replaced some uses");
1281 replaced = true;
1282 } else if (!phase->is_dominator(projs->fallthrough_catchproj, c)) {
1283 if (u->is_If()) {
1284 // Can't break If/Bool/Cmp chain
1285 assert(n->is_Bool(), "unexpected If shape");
1286 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1287 assert(n_clone->is_Bool(), "unexpected clone");
1288 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1289 Node* bol_clone = n->clone();
1290 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1291 bol_clone->set_req(1, cmp_clone);
1292
1293 Node* nn = stack.node_at(stack.size()-3);
1294 Node* nn_clone = clones.at(clones.size()-3);
1295 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1296
1297 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1298 &phase->igvn());
1299 assert(nb > 0, "should have replaced some uses");
1300
1301 phase->register_new_node(bol_clone, u->in(0));
1302 phase->register_new_node(cmp_clone, u->in(0));
1853 } else {
1854 assert(c != c->in(0), "");
1855 c = c->in(0);
1856 }
1857 }
1858 }
1859 }
1860 } while (stack.size() > 0);
1861 assert(mem != nullptr, "should have found safepoint");
1862 } else {
1863 mem = phi_mem;
1864 }
1865 return mem;
1866 }
1867
1868 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
1869 Node* c = _phase->get_ctrl(n);
1870 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
1871 assert(c == n->in(0), "");
1872 CallNode* call = c->as_Call();
1873 CallProjections* projs = call->extract_projections(true, false);
1874 if (projs->catchall_memproj != nullptr) {
1875 if (projs->fallthrough_memproj == n) {
1876 c = projs->fallthrough_catchproj;
1877 } else {
1878 assert(projs->catchall_memproj == n, "");
1879 c = projs->catchall_catchproj;
1880 }
1881 }
1882 }
1883 return c;
1884 }
1885
1886 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
1887 if (_phase->has_ctrl(n))
1888 return get_ctrl(n);
1889 else {
1890 assert (n->is_CFG(), "must be a CFG node");
1891 return n;
1892 }
1893 }
1894
1895 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
1896 return m != nullptr && get_ctrl(m) == c;
1897 }
1898
1899 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|