436 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
437 "sha512_implCompressMB",
438 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
439 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
440 "encodeBlock",
441 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
442 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
443 "decodeBlock",
444 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
445 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
446 "intpoly_montgomeryMult_P256",
447 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahStore },
448 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
449 "intpoly_assign",
450 { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone },
451 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
452 };
453
454 if (call->is_call_to_arraycopystub()) {
455 Node* dest = nullptr;
456 const TypeTuple* args = n->as_Call()->_tf->domain();
457 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
458 if (args->field_at(i)->isa_ptr()) {
459 j++;
460 if (j == 2) {
461 dest = n->in(i);
462 break;
463 }
464 }
465 }
466 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
467 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
468 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
469 }
470 } else if (strlen(call->_name) > 5 &&
471 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
472 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
473 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
474 }
475 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
476 // skip
557 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
558 Op_VectorizedHashCode,
559 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
560 Op_EncodeISOArray,
561 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
562 Op_CountPositives,
563 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
564 Op_CastP2X,
565 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
566 Op_StrIndexOfChar,
567 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
568 };
569
570 const int others_len = sizeof(others) / sizeof(others[0]);
571 int i = 0;
572 for (; i < others_len; i++) {
573 if (others[i].opcode == n->Opcode()) {
574 break;
575 }
576 }
577 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
578 if (i != others_len) {
579 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
580 for (uint j = 0; j < inputs_len; j++) {
581 int pos = others[i].inputs[j].pos;
582 if (pos == -1) {
583 break;
584 }
585 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
586 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
587 }
588 }
589 for (uint j = 1; j < stop; j++) {
590 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
591 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
592 uint k = 0;
593 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
594 if (k == inputs_len) {
595 fatal("arg %d for node %s not covered", j, n->Name());
596 }
597 }
777 mem_ctrl = phase->ctrl_or_self(mem);
778 }
779 return mem;
780 }
781
782 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
783 Node* mem = nullptr;
784 Node* c = ctrl;
785 do {
786 if (c->is_Region()) {
787 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
788 Node* u = c->fast_out(i);
789 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
790 if (u->adr_type() == TypePtr::BOTTOM) {
791 mem = u;
792 }
793 }
794 }
795 } else {
796 if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
797 CallProjections projs;
798 c->as_Call()->extract_projections(&projs, true, false);
799 if (projs.fallthrough_memproj != nullptr) {
800 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
801 if (projs.catchall_memproj == nullptr) {
802 mem = projs.fallthrough_memproj;
803 } else {
804 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
805 mem = projs.fallthrough_memproj;
806 } else {
807 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
808 mem = projs.catchall_memproj;
809 }
810 }
811 }
812 } else {
813 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
814 if (proj != nullptr &&
815 proj->adr_type() == TypePtr::BOTTOM) {
816 mem = proj;
817 }
818 }
819 } else {
820 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
821 Node* u = c->fast_out(i);
822 if (u->is_Proj() &&
823 u->bottom_type() == Type::MEMORY &&
824 u->adr_type() == TypePtr::BOTTOM) {
825 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
826 assert(mem == nullptr, "only one proj");
827 mem = u;
828 }
1049 if (c != ctrl ||
1050 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1051 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1052 phase->igvn().rehash_node_delayed(u);
1053 int nb = u->replace_edge(ctrl, region, &phase->igvn());
1054 if (u->is_CFG()) {
1055 if (phase->idom(u) == ctrl) {
1056 phase->set_idom(u, region, phase->dom_depth(region));
1057 }
1058 } else if (phase->get_ctrl(u) == ctrl) {
1059 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1060 uses.push(u);
1061 }
1062 assert(nb == 1, "more than 1 ctrl input?");
1063 --i, imax -= nb;
1064 }
1065 }
1066 }
1067 }
1068
1069 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
1070 Node* region = nullptr;
1071 while (c != ctrl) {
1072 if (c->is_Region()) {
1073 region = c;
1074 }
1075 c = phase->idom(c);
1076 }
1077 assert(region != nullptr, "");
1078 Node* phi = new PhiNode(region, n->bottom_type());
1079 for (uint j = 1; j < region->req(); j++) {
1080 Node* in = region->in(j);
1081 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1082 phi->init_req(j, n);
1083 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1084 phi->init_req(j, n_clone);
1085 } else {
1086 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1087 }
1088 }
1089 phase->register_new_node(phi, region);
1090 return phi;
1091 }
1092
1093 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1094 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1095
1096 Unique_Node_List uses;
1097 Node_Stack stack(0);
1098 Node_List clones;
1099 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1100 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1101
1102 Node* ctrl = phase->get_ctrl(lrb);
1103 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1166 if (phase->has_ctrl(in)) {
1167 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1168 #ifdef ASSERT
1169 for (uint i = 0; i < stack.size(); i++) {
1170 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1171 }
1172 #endif
1173 stack.push(in, 0);
1174 }
1175 } else {
1176 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1177 }
1178 }
1179 } else {
1180 phase->set_ctrl(n, call->in(0));
1181 stack.pop();
1182 }
1183 } while(stack.size() > 0);
1184 continue;
1185 }
1186 CallProjections projs;
1187 call->extract_projections(&projs, false, false);
1188
1189 // If this is a runtime call, it doesn't have an exception handling path
1190 if (projs.fallthrough_catchproj == nullptr) {
1191 assert(call->method() == nullptr, "should be runtime call");
1192 assert(projs.catchall_catchproj == nullptr, "runtime call should not have catch all projection");
1193 continue;
1194 }
1195
1196 // Otherwise, clone the barrier so there's one for the fallthrough and one for the exception handling path
1197 #ifdef ASSERT
1198 VectorSet cloned;
1199 #endif
1200 Node* lrb_clone = lrb->clone();
1201 phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1202 phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1203
1204 stack.push(lrb, 0);
1205 clones.push(lrb_clone);
1206
1207 do {
1208 assert(stack.size() == clones.size(), "");
1209 Node* n = stack.node();
1210 #ifdef ASSERT
1211 if (n->is_Load()) {
1212 Node* mem = n->in(MemNode::Memory);
1213 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1214 Node* u = mem->fast_out(j);
1215 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1216 }
1217 }
1218 #endif
1219 uint idx = stack.index();
1220 Node* n_clone = clones.at(clones.size()-1);
1221 if (idx < n->outcnt()) {
1222 Node* u = n->raw_out(idx);
1223 Node* c = phase->ctrl_or_self(u);
1224 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) {
1225 stack.set_index(idx+1);
1226 assert(!u->is_CFG(), "");
1227 stack.push(u, 0);
1228 assert(!cloned.test_set(u->_idx), "only one clone");
1229 Node* u_clone = u->clone();
1230 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1231 assert(nb > 0, "should have replaced some uses");
1232 phase->register_new_node(u_clone, projs.catchall_catchproj);
1233 clones.push(u_clone);
1234 phase->set_ctrl(u, projs.fallthrough_catchproj);
1235 } else {
1236 bool replaced = false;
1237 if (u->is_Phi()) {
1238 for (uint k = 1; k < u->req(); k++) {
1239 if (u->in(k) == n) {
1240 if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1241 phase->igvn().replace_input_of(u, k, n_clone);
1242 replaced = true;
1243 } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1244 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1245 replaced = true;
1246 }
1247 }
1248 }
1249 } else {
1250 if (phase->is_dominator(projs.catchall_catchproj, c)) {
1251 phase->igvn().rehash_node_delayed(u);
1252 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1253 assert(nb > 0, "should have replaced some uses");
1254 replaced = true;
1255 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1256 if (u->is_If()) {
1257 // Can't break If/Bool/Cmp chain
1258 assert(n->is_Bool(), "unexpected If shape");
1259 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1260 assert(n_clone->is_Bool(), "unexpected clone");
1261 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1262 Node* bol_clone = n->clone();
1263 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1264 bol_clone->set_req(1, cmp_clone);
1265
1266 Node* nn = stack.node_at(stack.size()-3);
1267 Node* nn_clone = clones.at(clones.size()-3);
1268 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1269
1270 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1271 &phase->igvn());
1272 assert(nb > 0, "should have replaced some uses");
1273
1274 phase->register_new_node(bol_clone, u->in(0));
1275 phase->register_new_node(cmp_clone, u->in(0));
2056 } else {
2057 assert(c != c->in(0), "");
2058 c = c->in(0);
2059 }
2060 }
2061 }
2062 }
2063 } while (stack.size() > 0);
2064 assert(mem != nullptr, "should have found safepoint");
2065 } else {
2066 mem = phi_mem;
2067 }
2068 return mem;
2069 }
2070
2071 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
2072 Node* c = _phase->get_ctrl(n);
2073 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
2074 assert(c == n->in(0), "");
2075 CallNode* call = c->as_Call();
2076 CallProjections projs;
2077 call->extract_projections(&projs, true, false);
2078 if (projs.catchall_memproj != nullptr) {
2079 if (projs.fallthrough_memproj == n) {
2080 c = projs.fallthrough_catchproj;
2081 } else {
2082 assert(projs.catchall_memproj == n, "");
2083 c = projs.catchall_catchproj;
2084 }
2085 }
2086 }
2087 return c;
2088 }
2089
2090 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
2091 if (_phase->has_ctrl(n))
2092 return get_ctrl(n);
2093 else {
2094 assert (n->is_CFG(), "must be a CFG node");
2095 return n;
2096 }
2097 }
2098
2099 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2100 return m != nullptr && get_ctrl(m) == c;
2101 }
2102
2103 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|
436 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
437 "sha512_implCompressMB",
438 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
439 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
440 "encodeBlock",
441 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
442 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
443 "decodeBlock",
444 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
445 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
446 "intpoly_montgomeryMult_P256",
447 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahStore },
448 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
449 "intpoly_assign",
450 { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone },
451 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
452 };
453
454 if (call->is_call_to_arraycopystub()) {
455 Node* dest = nullptr;
456 const TypeTuple* args = n->as_Call()->_tf->domain_sig();
457 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
458 if (args->field_at(i)->isa_ptr()) {
459 j++;
460 if (j == 2) {
461 dest = n->in(i);
462 break;
463 }
464 }
465 }
466 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
467 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
468 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
469 }
470 } else if (strlen(call->_name) > 5 &&
471 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
472 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
473 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
474 }
475 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
476 // skip
557 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
558 Op_VectorizedHashCode,
559 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
560 Op_EncodeISOArray,
561 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
562 Op_CountPositives,
563 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
564 Op_CastP2X,
565 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
566 Op_StrIndexOfChar,
567 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
568 };
569
570 const int others_len = sizeof(others) / sizeof(others[0]);
571 int i = 0;
572 for (; i < others_len; i++) {
573 if (others[i].opcode == n->Opcode()) {
574 break;
575 }
576 }
577 uint stop = n->is_Call() ? n->as_Call()->tf()->domain_sig()->cnt() : n->req();
578 if (i != others_len) {
579 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
580 for (uint j = 0; j < inputs_len; j++) {
581 int pos = others[i].inputs[j].pos;
582 if (pos == -1) {
583 break;
584 }
585 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
586 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
587 }
588 }
589 for (uint j = 1; j < stop; j++) {
590 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
591 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
592 uint k = 0;
593 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
594 if (k == inputs_len) {
595 fatal("arg %d for node %s not covered", j, n->Name());
596 }
597 }
777 mem_ctrl = phase->ctrl_or_self(mem);
778 }
779 return mem;
780 }
781
782 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
783 Node* mem = nullptr;
784 Node* c = ctrl;
785 do {
786 if (c->is_Region()) {
787 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
788 Node* u = c->fast_out(i);
789 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
790 if (u->adr_type() == TypePtr::BOTTOM) {
791 mem = u;
792 }
793 }
794 }
795 } else {
796 if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
797 CallProjections* projs = c->as_Call()->extract_projections(true, false);
798 if (projs->fallthrough_memproj != nullptr) {
799 if (projs->fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
800 if (projs->catchall_memproj == nullptr) {
801 mem = projs->fallthrough_memproj;
802 } else {
803 if (phase->is_dominator(projs->fallthrough_catchproj, ctrl)) {
804 mem = projs->fallthrough_memproj;
805 } else {
806 assert(phase->is_dominator(projs->catchall_catchproj, ctrl), "one proj must dominate barrier");
807 mem = projs->catchall_memproj;
808 }
809 }
810 }
811 } else {
812 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
813 if (proj != nullptr &&
814 proj->adr_type() == TypePtr::BOTTOM) {
815 mem = proj;
816 }
817 }
818 } else {
819 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
820 Node* u = c->fast_out(i);
821 if (u->is_Proj() &&
822 u->bottom_type() == Type::MEMORY &&
823 u->adr_type() == TypePtr::BOTTOM) {
824 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
825 assert(mem == nullptr, "only one proj");
826 mem = u;
827 }
1048 if (c != ctrl ||
1049 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1050 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1051 phase->igvn().rehash_node_delayed(u);
1052 int nb = u->replace_edge(ctrl, region, &phase->igvn());
1053 if (u->is_CFG()) {
1054 if (phase->idom(u) == ctrl) {
1055 phase->set_idom(u, region, phase->dom_depth(region));
1056 }
1057 } else if (phase->get_ctrl(u) == ctrl) {
1058 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1059 uses.push(u);
1060 }
1061 assert(nb == 1, "more than 1 ctrl input?");
1062 --i, imax -= nb;
1063 }
1064 }
1065 }
1066 }
1067
1068 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections* projs, PhaseIdealLoop* phase) {
1069 Node* region = nullptr;
1070 while (c != ctrl) {
1071 if (c->is_Region()) {
1072 region = c;
1073 }
1074 c = phase->idom(c);
1075 }
1076 assert(region != nullptr, "");
1077 Node* phi = new PhiNode(region, n->bottom_type());
1078 for (uint j = 1; j < region->req(); j++) {
1079 Node* in = region->in(j);
1080 if (phase->is_dominator(projs->fallthrough_catchproj, in)) {
1081 phi->init_req(j, n);
1082 } else if (phase->is_dominator(projs->catchall_catchproj, in)) {
1083 phi->init_req(j, n_clone);
1084 } else {
1085 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1086 }
1087 }
1088 phase->register_new_node(phi, region);
1089 return phi;
1090 }
1091
1092 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1093 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1094
1095 Unique_Node_List uses;
1096 Node_Stack stack(0);
1097 Node_List clones;
1098 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1099 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1100
1101 Node* ctrl = phase->get_ctrl(lrb);
1102 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1165 if (phase->has_ctrl(in)) {
1166 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1167 #ifdef ASSERT
1168 for (uint i = 0; i < stack.size(); i++) {
1169 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1170 }
1171 #endif
1172 stack.push(in, 0);
1173 }
1174 } else {
1175 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1176 }
1177 }
1178 } else {
1179 phase->set_ctrl(n, call->in(0));
1180 stack.pop();
1181 }
1182 } while(stack.size() > 0);
1183 continue;
1184 }
1185 CallProjections* projs = call->extract_projections(false, false);
1186
1187 // If this is a runtime call, it doesn't have an exception handling path
1188 if (projs->fallthrough_catchproj == nullptr) {
1189 assert(call->method() == nullptr, "should be runtime call");
1190 assert(projs->catchall_catchproj == nullptr, "runtime call should not have catch all projection");
1191 continue;
1192 }
1193
1194 // Otherwise, clone the barrier so there's one for the fallthrough and one for the exception handling path
1195 #ifdef ASSERT
1196 VectorSet cloned;
1197 #endif
1198 Node* lrb_clone = lrb->clone();
1199 phase->register_new_node(lrb_clone, projs->catchall_catchproj);
1200 phase->set_ctrl(lrb, projs->fallthrough_catchproj);
1201
1202 stack.push(lrb, 0);
1203 clones.push(lrb_clone);
1204
1205 do {
1206 assert(stack.size() == clones.size(), "");
1207 Node* n = stack.node();
1208 #ifdef ASSERT
1209 if (n->is_Load()) {
1210 Node* mem = n->in(MemNode::Memory);
1211 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1212 Node* u = mem->fast_out(j);
1213 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1214 }
1215 }
1216 #endif
1217 uint idx = stack.index();
1218 Node* n_clone = clones.at(clones.size()-1);
1219 if (idx < n->outcnt()) {
1220 Node* u = n->raw_out(idx);
1221 Node* c = phase->ctrl_or_self(u);
1222 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs->fallthrough_proj)) {
1223 stack.set_index(idx+1);
1224 assert(!u->is_CFG(), "");
1225 stack.push(u, 0);
1226 assert(!cloned.test_set(u->_idx), "only one clone");
1227 Node* u_clone = u->clone();
1228 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1229 assert(nb > 0, "should have replaced some uses");
1230 phase->register_new_node(u_clone, projs->catchall_catchproj);
1231 clones.push(u_clone);
1232 phase->set_ctrl(u, projs->fallthrough_catchproj);
1233 } else {
1234 bool replaced = false;
1235 if (u->is_Phi()) {
1236 for (uint k = 1; k < u->req(); k++) {
1237 if (u->in(k) == n) {
1238 if (phase->is_dominator(projs->catchall_catchproj, u->in(0)->in(k))) {
1239 phase->igvn().replace_input_of(u, k, n_clone);
1240 replaced = true;
1241 } else if (!phase->is_dominator(projs->fallthrough_catchproj, u->in(0)->in(k))) {
1242 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1243 replaced = true;
1244 }
1245 }
1246 }
1247 } else {
1248 if (phase->is_dominator(projs->catchall_catchproj, c)) {
1249 phase->igvn().rehash_node_delayed(u);
1250 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1251 assert(nb > 0, "should have replaced some uses");
1252 replaced = true;
1253 } else if (!phase->is_dominator(projs->fallthrough_catchproj, c)) {
1254 if (u->is_If()) {
1255 // Can't break If/Bool/Cmp chain
1256 assert(n->is_Bool(), "unexpected If shape");
1257 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1258 assert(n_clone->is_Bool(), "unexpected clone");
1259 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1260 Node* bol_clone = n->clone();
1261 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1262 bol_clone->set_req(1, cmp_clone);
1263
1264 Node* nn = stack.node_at(stack.size()-3);
1265 Node* nn_clone = clones.at(clones.size()-3);
1266 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1267
1268 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1269 &phase->igvn());
1270 assert(nb > 0, "should have replaced some uses");
1271
1272 phase->register_new_node(bol_clone, u->in(0));
1273 phase->register_new_node(cmp_clone, u->in(0));
2054 } else {
2055 assert(c != c->in(0), "");
2056 c = c->in(0);
2057 }
2058 }
2059 }
2060 }
2061 } while (stack.size() > 0);
2062 assert(mem != nullptr, "should have found safepoint");
2063 } else {
2064 mem = phi_mem;
2065 }
2066 return mem;
2067 }
2068
2069 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
2070 Node* c = _phase->get_ctrl(n);
2071 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
2072 assert(c == n->in(0), "");
2073 CallNode* call = c->as_Call();
2074 CallProjections* projs = call->extract_projections(true, false);
2075 if (projs->catchall_memproj != nullptr) {
2076 if (projs->fallthrough_memproj == n) {
2077 c = projs->fallthrough_catchproj;
2078 } else {
2079 assert(projs->catchall_memproj == n, "");
2080 c = projs->catchall_catchproj;
2081 }
2082 }
2083 }
2084 return c;
2085 }
2086
2087 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
2088 if (_phase->has_ctrl(n))
2089 return get_ctrl(n);
2090 else {
2091 assert (n->is_CFG(), "must be a CFG node");
2092 return n;
2093 }
2094 }
2095
2096 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2097 return m != nullptr && get_ctrl(m) == c;
2098 }
2099
2100 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|