430 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
431 "sha512_implCompressMB",
432 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
433 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
434 "encodeBlock",
435 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
436 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
437 "decodeBlock",
438 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
439 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
440 "intpoly_montgomeryMult_P256",
441 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahStore },
442 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
443 "intpoly_assign",
444 { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone },
445 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
446 };
447
448 if (call->is_call_to_arraycopystub()) {
449 Node* dest = nullptr;
450 const TypeTuple* args = n->as_Call()->_tf->domain();
451 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
452 if (args->field_at(i)->isa_ptr()) {
453 j++;
454 if (j == 2) {
455 dest = n->in(i);
456 break;
457 }
458 }
459 }
460 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
461 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
462 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
463 }
464 } else if (strlen(call->_name) > 5 &&
465 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
466 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
467 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
468 }
469 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
470 // skip
551 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
552 Op_VectorizedHashCode,
553 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
554 Op_EncodeISOArray,
555 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
556 Op_CountPositives,
557 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
558 Op_CastP2X,
559 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
560 Op_StrIndexOfChar,
561 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
562 };
563
564 const int others_len = sizeof(others) / sizeof(others[0]);
565 int i = 0;
566 for (; i < others_len; i++) {
567 if (others[i].opcode == n->Opcode()) {
568 break;
569 }
570 }
571 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
572 if (i != others_len) {
573 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
574 for (uint j = 0; j < inputs_len; j++) {
575 int pos = others[i].inputs[j].pos;
576 if (pos == -1) {
577 break;
578 }
579 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
580 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
581 }
582 }
583 for (uint j = 1; j < stop; j++) {
584 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
585 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
586 uint k = 0;
587 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
588 if (k == inputs_len) {
589 fatal("arg %d for node %s not covered", j, n->Name());
590 }
591 }
771 mem_ctrl = phase->ctrl_or_self(mem);
772 }
773 return mem;
774 }
775
776 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
777 Node* mem = nullptr;
778 Node* c = ctrl;
779 do {
780 if (c->is_Region()) {
781 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
782 Node* u = c->fast_out(i);
783 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
784 if (u->adr_type() == TypePtr::BOTTOM) {
785 mem = u;
786 }
787 }
788 }
789 } else {
790 if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
791 CallProjections projs;
792 c->as_Call()->extract_projections(&projs, true, false);
793 if (projs.fallthrough_memproj != nullptr) {
794 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
795 if (projs.catchall_memproj == nullptr) {
796 mem = projs.fallthrough_memproj;
797 } else {
798 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
799 mem = projs.fallthrough_memproj;
800 } else {
801 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
802 mem = projs.catchall_memproj;
803 }
804 }
805 }
806 } else {
807 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
808 if (proj != nullptr &&
809 proj->adr_type() == TypePtr::BOTTOM) {
810 mem = proj;
811 }
812 }
813 } else {
814 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
815 Node* u = c->fast_out(i);
816 if (u->is_Proj() &&
817 u->bottom_type() == Type::MEMORY &&
818 u->adr_type() == TypePtr::BOTTOM) {
819 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
820 assert(mem == nullptr, "only one proj");
821 mem = u;
822 }
1043 if (c != ctrl ||
1044 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1045 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1046 phase->igvn().rehash_node_delayed(u);
1047 int nb = u->replace_edge(ctrl, region, &phase->igvn());
1048 if (u->is_CFG()) {
1049 if (phase->idom(u) == ctrl) {
1050 phase->set_idom(u, region, phase->dom_depth(region));
1051 }
1052 } else if (phase->get_ctrl(u) == ctrl) {
1053 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1054 uses.push(u);
1055 }
1056 assert(nb == 1, "more than 1 ctrl input?");
1057 --i, imax -= nb;
1058 }
1059 }
1060 }
1061 }
1062
1063 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
1064 Node* region = nullptr;
1065 while (c != ctrl) {
1066 if (c->is_Region()) {
1067 region = c;
1068 }
1069 c = phase->idom(c);
1070 }
1071 assert(region != nullptr, "");
1072 Node* phi = new PhiNode(region, n->bottom_type());
1073 for (uint j = 1; j < region->req(); j++) {
1074 Node* in = region->in(j);
1075 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1076 phi->init_req(j, n);
1077 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1078 phi->init_req(j, n_clone);
1079 } else {
1080 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1081 }
1082 }
1083 phase->register_new_node(phi, region);
1084 return phi;
1085 }
1086
1087 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1088 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1089
1090 Unique_Node_List uses;
1091 Node_Stack stack(0);
1092 Node_List clones;
1093 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1094 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1095
1096 Node* ctrl = phase->get_ctrl(lrb);
1097 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1160 if (phase->has_ctrl(in)) {
1161 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1162 #ifdef ASSERT
1163 for (uint i = 0; i < stack.size(); i++) {
1164 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1165 }
1166 #endif
1167 stack.push(in, 0);
1168 }
1169 } else {
1170 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1171 }
1172 }
1173 } else {
1174 phase->set_ctrl(n, call->in(0));
1175 stack.pop();
1176 }
1177 } while(stack.size() > 0);
1178 continue;
1179 }
1180 CallProjections projs;
1181 call->extract_projections(&projs, false, false);
1182
1183 // If this is a runtime call, it doesn't have an exception handling path
1184 if (projs.fallthrough_catchproj == nullptr) {
1185 assert(call->method() == nullptr, "should be runtime call");
1186 assert(projs.catchall_catchproj == nullptr, "runtime call should not have catch all projection");
1187 continue;
1188 }
1189
1190 // Otherwise, clone the barrier so there's one for the fallthrough and one for the exception handling path
1191 #ifdef ASSERT
1192 VectorSet cloned;
1193 #endif
1194 Node* lrb_clone = lrb->clone();
1195 phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1196 phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1197
1198 stack.push(lrb, 0);
1199 clones.push(lrb_clone);
1200
1201 do {
1202 assert(stack.size() == clones.size(), "");
1203 Node* n = stack.node();
1204 #ifdef ASSERT
1205 if (n->is_Load()) {
1206 Node* mem = n->in(MemNode::Memory);
1207 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1208 Node* u = mem->fast_out(j);
1209 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1210 }
1211 }
1212 #endif
1213 uint idx = stack.index();
1214 Node* n_clone = clones.at(clones.size()-1);
1215 if (idx < n->outcnt()) {
1216 Node* u = n->raw_out(idx);
1217 Node* c = phase->ctrl_or_self(u);
1218 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) {
1219 stack.set_index(idx+1);
1220 assert(!u->is_CFG(), "");
1221 stack.push(u, 0);
1222 assert(!cloned.test_set(u->_idx), "only one clone");
1223 Node* u_clone = u->clone();
1224 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1225 assert(nb > 0, "should have replaced some uses");
1226 phase->register_new_node(u_clone, projs.catchall_catchproj);
1227 clones.push(u_clone);
1228 phase->set_ctrl(u, projs.fallthrough_catchproj);
1229 } else {
1230 bool replaced = false;
1231 if (u->is_Phi()) {
1232 for (uint k = 1; k < u->req(); k++) {
1233 if (u->in(k) == n) {
1234 if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1235 phase->igvn().replace_input_of(u, k, n_clone);
1236 replaced = true;
1237 } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1238 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1239 replaced = true;
1240 }
1241 }
1242 }
1243 } else {
1244 if (phase->is_dominator(projs.catchall_catchproj, c)) {
1245 phase->igvn().rehash_node_delayed(u);
1246 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1247 assert(nb > 0, "should have replaced some uses");
1248 replaced = true;
1249 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1250 if (u->is_If()) {
1251 // Can't break If/Bool/Cmp chain
1252 assert(n->is_Bool(), "unexpected If shape");
1253 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1254 assert(n_clone->is_Bool(), "unexpected clone");
1255 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1256 Node* bol_clone = n->clone();
1257 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1258 bol_clone->set_req(1, cmp_clone);
1259
1260 Node* nn = stack.node_at(stack.size()-3);
1261 Node* nn_clone = clones.at(clones.size()-3);
1262 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1263
1264 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1265 &phase->igvn());
1266 assert(nb > 0, "should have replaced some uses");
1267
1268 phase->register_new_node(bol_clone, u->in(0));
1269 phase->register_new_node(cmp_clone, u->in(0));
1820 } else {
1821 assert(c != c->in(0), "");
1822 c = c->in(0);
1823 }
1824 }
1825 }
1826 }
1827 } while (stack.size() > 0);
1828 assert(mem != nullptr, "should have found safepoint");
1829 } else {
1830 mem = phi_mem;
1831 }
1832 return mem;
1833 }
1834
1835 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
1836 Node* c = _phase->get_ctrl(n);
1837 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
1838 assert(c == n->in(0), "");
1839 CallNode* call = c->as_Call();
1840 CallProjections projs;
1841 call->extract_projections(&projs, true, false);
1842 if (projs.catchall_memproj != nullptr) {
1843 if (projs.fallthrough_memproj == n) {
1844 c = projs.fallthrough_catchproj;
1845 } else {
1846 assert(projs.catchall_memproj == n, "");
1847 c = projs.catchall_catchproj;
1848 }
1849 }
1850 }
1851 return c;
1852 }
1853
1854 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
1855 if (_phase->has_ctrl(n))
1856 return get_ctrl(n);
1857 else {
1858 assert (n->is_CFG(), "must be a CFG node");
1859 return n;
1860 }
1861 }
1862
1863 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
1864 return m != nullptr && get_ctrl(m) == c;
1865 }
1866
1867 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|
430 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
431 "sha512_implCompressMB",
432 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
433 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
434 "encodeBlock",
435 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
436 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
437 "decodeBlock",
438 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
439 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
440 "intpoly_montgomeryMult_P256",
441 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahStore },
442 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
443 "intpoly_assign",
444 { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone },
445 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
446 };
447
448 if (call->is_call_to_arraycopystub()) {
449 Node* dest = nullptr;
450 const TypeTuple* args = n->as_Call()->_tf->domain_sig();
451 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
452 if (args->field_at(i)->isa_ptr()) {
453 j++;
454 if (j == 2) {
455 dest = n->in(i);
456 break;
457 }
458 }
459 }
460 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
461 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
462 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
463 }
464 } else if (strlen(call->_name) > 5 &&
465 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
466 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
467 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
468 }
469 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
470 // skip
551 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
552 Op_VectorizedHashCode,
553 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
554 Op_EncodeISOArray,
555 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
556 Op_CountPositives,
557 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
558 Op_CastP2X,
559 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
560 Op_StrIndexOfChar,
561 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
562 };
563
564 const int others_len = sizeof(others) / sizeof(others[0]);
565 int i = 0;
566 for (; i < others_len; i++) {
567 if (others[i].opcode == n->Opcode()) {
568 break;
569 }
570 }
571 uint stop = n->is_Call() ? n->as_Call()->tf()->domain_sig()->cnt() : n->req();
572 if (i != others_len) {
573 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
574 for (uint j = 0; j < inputs_len; j++) {
575 int pos = others[i].inputs[j].pos;
576 if (pos == -1) {
577 break;
578 }
579 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
580 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
581 }
582 }
583 for (uint j = 1; j < stop; j++) {
584 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
585 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
586 uint k = 0;
587 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
588 if (k == inputs_len) {
589 fatal("arg %d for node %s not covered", j, n->Name());
590 }
591 }
771 mem_ctrl = phase->ctrl_or_self(mem);
772 }
773 return mem;
774 }
775
776 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
777 Node* mem = nullptr;
778 Node* c = ctrl;
779 do {
780 if (c->is_Region()) {
781 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
782 Node* u = c->fast_out(i);
783 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
784 if (u->adr_type() == TypePtr::BOTTOM) {
785 mem = u;
786 }
787 }
788 }
789 } else {
790 if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
791 CallProjections* projs = c->as_Call()->extract_projections(true, false);
792 if (projs->fallthrough_memproj != nullptr) {
793 if (projs->fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
794 if (projs->catchall_memproj == nullptr) {
795 mem = projs->fallthrough_memproj;
796 } else {
797 if (phase->is_dominator(projs->fallthrough_catchproj, ctrl)) {
798 mem = projs->fallthrough_memproj;
799 } else {
800 assert(phase->is_dominator(projs->catchall_catchproj, ctrl), "one proj must dominate barrier");
801 mem = projs->catchall_memproj;
802 }
803 }
804 }
805 } else {
806 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
807 if (proj != nullptr &&
808 proj->adr_type() == TypePtr::BOTTOM) {
809 mem = proj;
810 }
811 }
812 } else {
813 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
814 Node* u = c->fast_out(i);
815 if (u->is_Proj() &&
816 u->bottom_type() == Type::MEMORY &&
817 u->adr_type() == TypePtr::BOTTOM) {
818 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
819 assert(mem == nullptr, "only one proj");
820 mem = u;
821 }
1042 if (c != ctrl ||
1043 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1044 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1045 phase->igvn().rehash_node_delayed(u);
1046 int nb = u->replace_edge(ctrl, region, &phase->igvn());
1047 if (u->is_CFG()) {
1048 if (phase->idom(u) == ctrl) {
1049 phase->set_idom(u, region, phase->dom_depth(region));
1050 }
1051 } else if (phase->get_ctrl(u) == ctrl) {
1052 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1053 uses.push(u);
1054 }
1055 assert(nb == 1, "more than 1 ctrl input?");
1056 --i, imax -= nb;
1057 }
1058 }
1059 }
1060 }
1061
1062 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections* projs, PhaseIdealLoop* phase) {
1063 Node* region = nullptr;
1064 while (c != ctrl) {
1065 if (c->is_Region()) {
1066 region = c;
1067 }
1068 c = phase->idom(c);
1069 }
1070 assert(region != nullptr, "");
1071 Node* phi = new PhiNode(region, n->bottom_type());
1072 for (uint j = 1; j < region->req(); j++) {
1073 Node* in = region->in(j);
1074 if (phase->is_dominator(projs->fallthrough_catchproj, in)) {
1075 phi->init_req(j, n);
1076 } else if (phase->is_dominator(projs->catchall_catchproj, in)) {
1077 phi->init_req(j, n_clone);
1078 } else {
1079 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1080 }
1081 }
1082 phase->register_new_node(phi, region);
1083 return phi;
1084 }
1085
1086 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1087 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1088
1089 Unique_Node_List uses;
1090 Node_Stack stack(0);
1091 Node_List clones;
1092 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1093 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1094
1095 Node* ctrl = phase->get_ctrl(lrb);
1096 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1159 if (phase->has_ctrl(in)) {
1160 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1161 #ifdef ASSERT
1162 for (uint i = 0; i < stack.size(); i++) {
1163 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1164 }
1165 #endif
1166 stack.push(in, 0);
1167 }
1168 } else {
1169 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1170 }
1171 }
1172 } else {
1173 phase->set_ctrl(n, call->in(0));
1174 stack.pop();
1175 }
1176 } while(stack.size() > 0);
1177 continue;
1178 }
1179 CallProjections* projs = call->extract_projections(false, false);
1180
1181 // If this is a runtime call, it doesn't have an exception handling path
1182 if (projs->fallthrough_catchproj == nullptr) {
1183 assert(call->method() == nullptr, "should be runtime call");
1184 assert(projs->catchall_catchproj == nullptr, "runtime call should not have catch all projection");
1185 continue;
1186 }
1187
1188 // Otherwise, clone the barrier so there's one for the fallthrough and one for the exception handling path
1189 #ifdef ASSERT
1190 VectorSet cloned;
1191 #endif
1192 Node* lrb_clone = lrb->clone();
1193 phase->register_new_node(lrb_clone, projs->catchall_catchproj);
1194 phase->set_ctrl(lrb, projs->fallthrough_catchproj);
1195
1196 stack.push(lrb, 0);
1197 clones.push(lrb_clone);
1198
1199 do {
1200 assert(stack.size() == clones.size(), "");
1201 Node* n = stack.node();
1202 #ifdef ASSERT
1203 if (n->is_Load()) {
1204 Node* mem = n->in(MemNode::Memory);
1205 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1206 Node* u = mem->fast_out(j);
1207 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1208 }
1209 }
1210 #endif
1211 uint idx = stack.index();
1212 Node* n_clone = clones.at(clones.size()-1);
1213 if (idx < n->outcnt()) {
1214 Node* u = n->raw_out(idx);
1215 Node* c = phase->ctrl_or_self(u);
1216 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs->fallthrough_proj)) {
1217 stack.set_index(idx+1);
1218 assert(!u->is_CFG(), "");
1219 stack.push(u, 0);
1220 assert(!cloned.test_set(u->_idx), "only one clone");
1221 Node* u_clone = u->clone();
1222 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1223 assert(nb > 0, "should have replaced some uses");
1224 phase->register_new_node(u_clone, projs->catchall_catchproj);
1225 clones.push(u_clone);
1226 phase->set_ctrl(u, projs->fallthrough_catchproj);
1227 } else {
1228 bool replaced = false;
1229 if (u->is_Phi()) {
1230 for (uint k = 1; k < u->req(); k++) {
1231 if (u->in(k) == n) {
1232 if (phase->is_dominator(projs->catchall_catchproj, u->in(0)->in(k))) {
1233 phase->igvn().replace_input_of(u, k, n_clone);
1234 replaced = true;
1235 } else if (!phase->is_dominator(projs->fallthrough_catchproj, u->in(0)->in(k))) {
1236 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1237 replaced = true;
1238 }
1239 }
1240 }
1241 } else {
1242 if (phase->is_dominator(projs->catchall_catchproj, c)) {
1243 phase->igvn().rehash_node_delayed(u);
1244 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1245 assert(nb > 0, "should have replaced some uses");
1246 replaced = true;
1247 } else if (!phase->is_dominator(projs->fallthrough_catchproj, c)) {
1248 if (u->is_If()) {
1249 // Can't break If/Bool/Cmp chain
1250 assert(n->is_Bool(), "unexpected If shape");
1251 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1252 assert(n_clone->is_Bool(), "unexpected clone");
1253 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1254 Node* bol_clone = n->clone();
1255 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1256 bol_clone->set_req(1, cmp_clone);
1257
1258 Node* nn = stack.node_at(stack.size()-3);
1259 Node* nn_clone = clones.at(clones.size()-3);
1260 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1261
1262 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1263 &phase->igvn());
1264 assert(nb > 0, "should have replaced some uses");
1265
1266 phase->register_new_node(bol_clone, u->in(0));
1267 phase->register_new_node(cmp_clone, u->in(0));
1818 } else {
1819 assert(c != c->in(0), "");
1820 c = c->in(0);
1821 }
1822 }
1823 }
1824 }
1825 } while (stack.size() > 0);
1826 assert(mem != nullptr, "should have found safepoint");
1827 } else {
1828 mem = phi_mem;
1829 }
1830 return mem;
1831 }
1832
1833 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
1834 Node* c = _phase->get_ctrl(n);
1835 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
1836 assert(c == n->in(0), "");
1837 CallNode* call = c->as_Call();
1838 CallProjections* projs = call->extract_projections(true, false);
1839 if (projs->catchall_memproj != nullptr) {
1840 if (projs->fallthrough_memproj == n) {
1841 c = projs->fallthrough_catchproj;
1842 } else {
1843 assert(projs->catchall_memproj == n, "");
1844 c = projs->catchall_catchproj;
1845 }
1846 }
1847 }
1848 return c;
1849 }
1850
1851 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
1852 if (_phase->has_ctrl(n))
1853 return get_ctrl(n);
1854 else {
1855 assert (n->is_CFG(), "must be a CFG node");
1856 return n;
1857 }
1858 }
1859
1860 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
1861 return m != nullptr && get_ctrl(m) == c;
1862 }
1863
1864 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|