430 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
431 "sha512_implCompressMB",
432 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
433 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
434 "encodeBlock",
435 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
436 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
437 "decodeBlock",
438 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
439 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
440 "intpoly_montgomeryMult_P256",
441 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahStore },
442 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
443 "intpoly_assign",
444 { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone },
445 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
446 };
447
448 if (call->is_call_to_arraycopystub()) {
449 Node* dest = nullptr;
450 const TypeTuple* args = n->as_Call()->_tf->domain();
451 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
452 if (args->field_at(i)->isa_ptr()) {
453 j++;
454 if (j == 2) {
455 dest = n->in(i);
456 break;
457 }
458 }
459 }
460 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
461 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
462 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
463 }
464 } else if (strlen(call->_name) > 5 &&
465 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
466 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
467 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
468 }
469 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
470 // skip
551 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
552 Op_VectorizedHashCode,
553 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
554 Op_EncodeISOArray,
555 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
556 Op_CountPositives,
557 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
558 Op_CastP2X,
559 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
560 Op_StrIndexOfChar,
561 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
562 };
563
564 const int others_len = sizeof(others) / sizeof(others[0]);
565 int i = 0;
566 for (; i < others_len; i++) {
567 if (others[i].opcode == n->Opcode()) {
568 break;
569 }
570 }
571 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
572 if (i != others_len) {
573 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
574 for (uint j = 0; j < inputs_len; j++) {
575 int pos = others[i].inputs[j].pos;
576 if (pos == -1) {
577 break;
578 }
579 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
580 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
581 }
582 }
583 for (uint j = 1; j < stop; j++) {
584 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
585 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
586 uint k = 0;
587 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
588 if (k == inputs_len) {
589 fatal("arg %d for node %s not covered", j, n->Name());
590 }
591 }
786 mem_ctrl = phase->ctrl_or_self(mem);
787 }
788 return mem;
789 }
790
791 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
792 Node* mem = nullptr;
793 Node* c = ctrl;
794 do {
795 if (c->is_Region()) {
796 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
797 Node* u = c->fast_out(i);
798 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
799 if (u->adr_type() == TypePtr::BOTTOM) {
800 mem = u;
801 }
802 }
803 }
804 } else {
805 if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
806 CallProjections projs;
807 c->as_Call()->extract_projections(&projs, true, false);
808 if (projs.fallthrough_memproj != nullptr) {
809 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
810 if (projs.catchall_memproj == nullptr) {
811 mem = projs.fallthrough_memproj;
812 } else {
813 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
814 mem = projs.fallthrough_memproj;
815 } else {
816 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
817 mem = projs.catchall_memproj;
818 }
819 }
820 }
821 } else {
822 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
823 if (proj != nullptr &&
824 proj->adr_type() == TypePtr::BOTTOM) {
825 mem = proj;
826 }
827 }
828 } else {
829 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
830 Node* u = c->fast_out(i);
831 if (u->is_Proj() &&
832 u->bottom_type() == Type::MEMORY &&
833 u->adr_type() == TypePtr::BOTTOM) {
834 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
835 assert(mem == nullptr, "only one proj");
836 mem = u;
837 }
1058 if (old_c != ctrl ||
1059 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1060 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1061 phase->igvn().rehash_node_delayed(u);
1062 int nb = u->replace_edge(ctrl, region, &phase->igvn());
1063 if (u->is_CFG()) {
1064 if (phase->idom(u) == ctrl) {
1065 phase->set_idom(u, region, phase->dom_depth(region));
1066 }
1067 } else if (phase->get_ctrl(u) == ctrl) {
1068 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1069 uses.push(u);
1070 }
1071 assert(nb == 1, "more than 1 ctrl input?");
1072 --i, imax -= nb;
1073 }
1074 }
1075 }
1076 }
1077
1078 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
1079 Node* region = nullptr;
1080 while (c != ctrl) {
1081 if (c->is_Region()) {
1082 region = c;
1083 }
1084 c = phase->idom(c);
1085 }
1086 assert(region != nullptr, "");
1087 Node* phi = new PhiNode(region, n->bottom_type());
1088 for (uint j = 1; j < region->req(); j++) {
1089 Node* in = region->in(j);
1090 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1091 phi->init_req(j, n);
1092 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1093 phi->init_req(j, n_clone);
1094 } else {
1095 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1096 }
1097 }
1098 phase->register_new_node(phi, region);
1099 return phi;
1100 }
1101
1102 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1103 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1104
1105 Unique_Node_List uses;
1106 Node_Stack stack(0);
1107 Node_List clones;
1108 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1109 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1110
1111 Node* ctrl = phase->get_ctrl(lrb);
1112 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1175 if (phase->has_ctrl(in)) {
1176 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1177 #ifdef ASSERT
1178 for (uint i = 0; i < stack.size(); i++) {
1179 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1180 }
1181 #endif
1182 stack.push(in, 0);
1183 }
1184 } else {
1185 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1186 }
1187 }
1188 } else {
1189 phase->set_ctrl(n, call->in(0));
1190 stack.pop();
1191 }
1192 } while(stack.size() > 0);
1193 continue;
1194 }
1195 CallProjections projs;
1196 call->extract_projections(&projs, false, false);
1197
1198 // If this is a runtime call, it doesn't have an exception handling path
1199 if (projs.fallthrough_catchproj == nullptr) {
1200 assert(call->method() == nullptr, "should be runtime call");
1201 assert(projs.catchall_catchproj == nullptr, "runtime call should not have catch all projection");
1202 continue;
1203 }
1204
1205 // Otherwise, clone the barrier so there's one for the fallthrough and one for the exception handling path
1206 #ifdef ASSERT
1207 VectorSet cloned;
1208 #endif
1209 Node* lrb_clone = lrb->clone();
1210 phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1211 phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1212
1213 stack.push(lrb, 0);
1214 clones.push(lrb_clone);
1215
1216 do {
1217 assert(stack.size() == clones.size(), "");
1218 Node* n = stack.node();
1219 #ifdef ASSERT
1220 if (n->is_Load()) {
1221 Node* mem = n->in(MemNode::Memory);
1222 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1223 Node* u = mem->fast_out(j);
1224 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1225 }
1226 }
1227 #endif
1228 uint idx = stack.index();
1229 Node* n_clone = clones.at(clones.size()-1);
1230 if (idx < n->outcnt()) {
1231 Node* u = n->raw_out(idx);
1232 Node* c = phase->ctrl_or_self(u);
1233 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) {
1234 stack.set_index(idx+1);
1235 assert(!u->is_CFG(), "");
1236 stack.push(u, 0);
1237 assert(!cloned.test_set(u->_idx), "only one clone");
1238 Node* u_clone = u->clone();
1239 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1240 assert(nb > 0, "should have replaced some uses");
1241 phase->register_new_node(u_clone, projs.catchall_catchproj);
1242 clones.push(u_clone);
1243 phase->set_ctrl(u, projs.fallthrough_catchproj);
1244 } else {
1245 bool replaced = false;
1246 if (u->is_Phi()) {
1247 for (uint k = 1; k < u->req(); k++) {
1248 if (u->in(k) == n) {
1249 if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1250 phase->igvn().replace_input_of(u, k, n_clone);
1251 replaced = true;
1252 } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1253 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1254 replaced = true;
1255 }
1256 }
1257 }
1258 } else {
1259 if (phase->is_dominator(projs.catchall_catchproj, c)) {
1260 phase->igvn().rehash_node_delayed(u);
1261 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1262 assert(nb > 0, "should have replaced some uses");
1263 replaced = true;
1264 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1265 if (u->is_If()) {
1266 // Can't break If/Bool/Cmp chain
1267 assert(n->is_Bool(), "unexpected If shape");
1268 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1269 assert(n_clone->is_Bool(), "unexpected clone");
1270 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1271 Node* bol_clone = n->clone();
1272 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1273 bol_clone->set_req(1, cmp_clone);
1274
1275 Node* nn = stack.node_at(stack.size()-3);
1276 Node* nn_clone = clones.at(clones.size()-3);
1277 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1278
1279 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1280 &phase->igvn());
1281 assert(nb > 0, "should have replaced some uses");
1282
1283 phase->register_new_node(bol_clone, u->in(0));
1284 phase->register_new_node(cmp_clone, u->in(0));
1835 } else {
1836 assert(c != c->in(0), "");
1837 c = c->in(0);
1838 }
1839 }
1840 }
1841 }
1842 } while (stack.size() > 0);
1843 assert(mem != nullptr, "should have found safepoint");
1844 } else {
1845 mem = phi_mem;
1846 }
1847 return mem;
1848 }
1849
1850 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
1851 Node* c = _phase->get_ctrl(n);
1852 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
1853 assert(c == n->in(0), "");
1854 CallNode* call = c->as_Call();
1855 CallProjections projs;
1856 call->extract_projections(&projs, true, false);
1857 if (projs.catchall_memproj != nullptr) {
1858 if (projs.fallthrough_memproj == n) {
1859 c = projs.fallthrough_catchproj;
1860 } else {
1861 assert(projs.catchall_memproj == n, "");
1862 c = projs.catchall_catchproj;
1863 }
1864 }
1865 }
1866 return c;
1867 }
1868
1869 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
1870 if (_phase->has_ctrl(n))
1871 return get_ctrl(n);
1872 else {
1873 assert (n->is_CFG(), "must be a CFG node");
1874 return n;
1875 }
1876 }
1877
1878 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
1879 return m != nullptr && get_ctrl(m) == c;
1880 }
1881
1882 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|
430 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
431 "sha512_implCompressMB",
432 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
433 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
434 "encodeBlock",
435 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
436 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
437 "decodeBlock",
438 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
439 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
440 "intpoly_montgomeryMult_P256",
441 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahStore },
442 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
443 "intpoly_assign",
444 { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone },
445 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
446 };
447
448 if (call->is_call_to_arraycopystub()) {
449 Node* dest = nullptr;
450 const TypeTuple* args = n->as_Call()->_tf->domain_sig();
451 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
452 if (args->field_at(i)->isa_ptr()) {
453 j++;
454 if (j == 2) {
455 dest = n->in(i);
456 break;
457 }
458 }
459 }
460 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
461 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
462 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
463 }
464 } else if (strlen(call->_name) > 5 &&
465 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
466 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
467 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
468 }
469 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
470 // skip
551 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
552 Op_VectorizedHashCode,
553 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
554 Op_EncodeISOArray,
555 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
556 Op_CountPositives,
557 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
558 Op_CastP2X,
559 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
560 Op_StrIndexOfChar,
561 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
562 };
563
564 const int others_len = sizeof(others) / sizeof(others[0]);
565 int i = 0;
566 for (; i < others_len; i++) {
567 if (others[i].opcode == n->Opcode()) {
568 break;
569 }
570 }
571 uint stop = n->is_Call() ? n->as_Call()->tf()->domain_sig()->cnt() : n->req();
572 if (i != others_len) {
573 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
574 for (uint j = 0; j < inputs_len; j++) {
575 int pos = others[i].inputs[j].pos;
576 if (pos == -1) {
577 break;
578 }
579 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
580 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
581 }
582 }
583 for (uint j = 1; j < stop; j++) {
584 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
585 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
586 uint k = 0;
587 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
588 if (k == inputs_len) {
589 fatal("arg %d for node %s not covered", j, n->Name());
590 }
591 }
786 mem_ctrl = phase->ctrl_or_self(mem);
787 }
788 return mem;
789 }
790
791 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
792 Node* mem = nullptr;
793 Node* c = ctrl;
794 do {
795 if (c->is_Region()) {
796 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
797 Node* u = c->fast_out(i);
798 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
799 if (u->adr_type() == TypePtr::BOTTOM) {
800 mem = u;
801 }
802 }
803 }
804 } else {
805 if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
806 CallProjections* projs = c->as_Call()->extract_projections(true, false);
807 if (projs->fallthrough_memproj != nullptr) {
808 if (projs->fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
809 if (projs->catchall_memproj == nullptr) {
810 mem = projs->fallthrough_memproj;
811 } else {
812 if (phase->is_dominator(projs->fallthrough_catchproj, ctrl)) {
813 mem = projs->fallthrough_memproj;
814 } else {
815 assert(phase->is_dominator(projs->catchall_catchproj, ctrl), "one proj must dominate barrier");
816 mem = projs->catchall_memproj;
817 }
818 }
819 }
820 } else {
821 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
822 if (proj != nullptr &&
823 proj->adr_type() == TypePtr::BOTTOM) {
824 mem = proj;
825 }
826 }
827 } else {
828 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
829 Node* u = c->fast_out(i);
830 if (u->is_Proj() &&
831 u->bottom_type() == Type::MEMORY &&
832 u->adr_type() == TypePtr::BOTTOM) {
833 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
834 assert(mem == nullptr, "only one proj");
835 mem = u;
836 }
1057 if (old_c != ctrl ||
1058 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1059 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1060 phase->igvn().rehash_node_delayed(u);
1061 int nb = u->replace_edge(ctrl, region, &phase->igvn());
1062 if (u->is_CFG()) {
1063 if (phase->idom(u) == ctrl) {
1064 phase->set_idom(u, region, phase->dom_depth(region));
1065 }
1066 } else if (phase->get_ctrl(u) == ctrl) {
1067 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1068 uses.push(u);
1069 }
1070 assert(nb == 1, "more than 1 ctrl input?");
1071 --i, imax -= nb;
1072 }
1073 }
1074 }
1075 }
1076
1077 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections* projs, PhaseIdealLoop* phase) {
1078 Node* region = nullptr;
1079 while (c != ctrl) {
1080 if (c->is_Region()) {
1081 region = c;
1082 }
1083 c = phase->idom(c);
1084 }
1085 assert(region != nullptr, "");
1086 Node* phi = new PhiNode(region, n->bottom_type());
1087 for (uint j = 1; j < region->req(); j++) {
1088 Node* in = region->in(j);
1089 if (phase->is_dominator(projs->fallthrough_catchproj, in)) {
1090 phi->init_req(j, n);
1091 } else if (phase->is_dominator(projs->catchall_catchproj, in)) {
1092 phi->init_req(j, n_clone);
1093 } else {
1094 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1095 }
1096 }
1097 phase->register_new_node(phi, region);
1098 return phi;
1099 }
1100
1101 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1102 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1103
1104 Unique_Node_List uses;
1105 Node_Stack stack(0);
1106 Node_List clones;
1107 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1108 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1109
1110 Node* ctrl = phase->get_ctrl(lrb);
1111 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1174 if (phase->has_ctrl(in)) {
1175 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1176 #ifdef ASSERT
1177 for (uint i = 0; i < stack.size(); i++) {
1178 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1179 }
1180 #endif
1181 stack.push(in, 0);
1182 }
1183 } else {
1184 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1185 }
1186 }
1187 } else {
1188 phase->set_ctrl(n, call->in(0));
1189 stack.pop();
1190 }
1191 } while(stack.size() > 0);
1192 continue;
1193 }
1194 CallProjections* projs = call->extract_projections(false, false);
1195
1196 // If this is a runtime call, it doesn't have an exception handling path
1197 if (projs->fallthrough_catchproj == nullptr) {
1198 assert(call->method() == nullptr, "should be runtime call");
1199 assert(projs->catchall_catchproj == nullptr, "runtime call should not have catch all projection");
1200 continue;
1201 }
1202
1203 // Otherwise, clone the barrier so there's one for the fallthrough and one for the exception handling path
1204 #ifdef ASSERT
1205 VectorSet cloned;
1206 #endif
1207 Node* lrb_clone = lrb->clone();
1208 phase->register_new_node(lrb_clone, projs->catchall_catchproj);
1209 phase->set_ctrl(lrb, projs->fallthrough_catchproj);
1210
1211 stack.push(lrb, 0);
1212 clones.push(lrb_clone);
1213
1214 do {
1215 assert(stack.size() == clones.size(), "");
1216 Node* n = stack.node();
1217 #ifdef ASSERT
1218 if (n->is_Load()) {
1219 Node* mem = n->in(MemNode::Memory);
1220 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1221 Node* u = mem->fast_out(j);
1222 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1223 }
1224 }
1225 #endif
1226 uint idx = stack.index();
1227 Node* n_clone = clones.at(clones.size()-1);
1228 if (idx < n->outcnt()) {
1229 Node* u = n->raw_out(idx);
1230 Node* c = phase->ctrl_or_self(u);
1231 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs->fallthrough_proj)) {
1232 stack.set_index(idx+1);
1233 assert(!u->is_CFG(), "");
1234 stack.push(u, 0);
1235 assert(!cloned.test_set(u->_idx), "only one clone");
1236 Node* u_clone = u->clone();
1237 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1238 assert(nb > 0, "should have replaced some uses");
1239 phase->register_new_node(u_clone, projs->catchall_catchproj);
1240 clones.push(u_clone);
1241 phase->set_ctrl(u, projs->fallthrough_catchproj);
1242 } else {
1243 bool replaced = false;
1244 if (u->is_Phi()) {
1245 for (uint k = 1; k < u->req(); k++) {
1246 if (u->in(k) == n) {
1247 if (phase->is_dominator(projs->catchall_catchproj, u->in(0)->in(k))) {
1248 phase->igvn().replace_input_of(u, k, n_clone);
1249 replaced = true;
1250 } else if (!phase->is_dominator(projs->fallthrough_catchproj, u->in(0)->in(k))) {
1251 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1252 replaced = true;
1253 }
1254 }
1255 }
1256 } else {
1257 if (phase->is_dominator(projs->catchall_catchproj, c)) {
1258 phase->igvn().rehash_node_delayed(u);
1259 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1260 assert(nb > 0, "should have replaced some uses");
1261 replaced = true;
1262 } else if (!phase->is_dominator(projs->fallthrough_catchproj, c)) {
1263 if (u->is_If()) {
1264 // Can't break If/Bool/Cmp chain
1265 assert(n->is_Bool(), "unexpected If shape");
1266 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1267 assert(n_clone->is_Bool(), "unexpected clone");
1268 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1269 Node* bol_clone = n->clone();
1270 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1271 bol_clone->set_req(1, cmp_clone);
1272
1273 Node* nn = stack.node_at(stack.size()-3);
1274 Node* nn_clone = clones.at(clones.size()-3);
1275 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1276
1277 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1278 &phase->igvn());
1279 assert(nb > 0, "should have replaced some uses");
1280
1281 phase->register_new_node(bol_clone, u->in(0));
1282 phase->register_new_node(cmp_clone, u->in(0));
1833 } else {
1834 assert(c != c->in(0), "");
1835 c = c->in(0);
1836 }
1837 }
1838 }
1839 }
1840 } while (stack.size() > 0);
1841 assert(mem != nullptr, "should have found safepoint");
1842 } else {
1843 mem = phi_mem;
1844 }
1845 return mem;
1846 }
1847
1848 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
1849 Node* c = _phase->get_ctrl(n);
1850 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
1851 assert(c == n->in(0), "");
1852 CallNode* call = c->as_Call();
1853 CallProjections* projs = call->extract_projections(true, false);
1854 if (projs->catchall_memproj != nullptr) {
1855 if (projs->fallthrough_memproj == n) {
1856 c = projs->fallthrough_catchproj;
1857 } else {
1858 assert(projs->catchall_memproj == n, "");
1859 c = projs->catchall_catchproj;
1860 }
1861 }
1862 }
1863 return c;
1864 }
1865
1866 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
1867 if (_phase->has_ctrl(n))
1868 return get_ctrl(n);
1869 else {
1870 assert (n->is_CFG(), "must be a CFG node");
1871 return n;
1872 }
1873 }
1874
1875 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
1876 return m != nullptr && get_ctrl(m) == c;
1877 }
1878
1879 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|