437 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
438 "sha512_implCompressMB",
439 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
440 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
441 "encodeBlock",
442 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
443 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
444 "decodeBlock",
445 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
446 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
447 "intpoly_montgomeryMult_P256",
448 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahStore },
449 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
450 "intpoly_assign",
451 { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone },
452 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
453 };
454
455 if (call->is_call_to_arraycopystub()) {
456 Node* dest = nullptr;
457 const TypeTuple* args = n->as_Call()->_tf->domain();
458 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
459 if (args->field_at(i)->isa_ptr()) {
460 j++;
461 if (j == 2) {
462 dest = n->in(i);
463 break;
464 }
465 }
466 }
467 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
468 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
469 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
470 }
471 } else if (strlen(call->_name) > 5 &&
472 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
473 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
474 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
475 }
476 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
477 // skip
558 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
559 Op_VectorizedHashCode,
560 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
561 Op_EncodeISOArray,
562 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
563 Op_CountPositives,
564 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
565 Op_CastP2X,
566 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
567 Op_StrIndexOfChar,
568 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
569 };
570
571 const int others_len = sizeof(others) / sizeof(others[0]);
572 int i = 0;
573 for (; i < others_len; i++) {
574 if (others[i].opcode == n->Opcode()) {
575 break;
576 }
577 }
578 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
579 if (i != others_len) {
580 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
581 for (uint j = 0; j < inputs_len; j++) {
582 int pos = others[i].inputs[j].pos;
583 if (pos == -1) {
584 break;
585 }
586 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
587 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
588 }
589 }
590 for (uint j = 1; j < stop; j++) {
591 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
592 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
593 uint k = 0;
594 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
595 if (k == inputs_len) {
596 fatal("arg %d for node %s not covered", j, n->Name());
597 }
598 }
778 mem_ctrl = phase->ctrl_or_self(mem);
779 }
780 return mem;
781 }
782
783 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
784 Node* mem = nullptr;
785 Node* c = ctrl;
786 do {
787 if (c->is_Region()) {
788 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
789 Node* u = c->fast_out(i);
790 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
791 if (u->adr_type() == TypePtr::BOTTOM) {
792 mem = u;
793 }
794 }
795 }
796 } else {
797 if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
798 CallProjections projs;
799 c->as_Call()->extract_projections(&projs, true, false);
800 if (projs.fallthrough_memproj != nullptr) {
801 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
802 if (projs.catchall_memproj == nullptr) {
803 mem = projs.fallthrough_memproj;
804 } else {
805 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
806 mem = projs.fallthrough_memproj;
807 } else {
808 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
809 mem = projs.catchall_memproj;
810 }
811 }
812 }
813 } else {
814 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
815 if (proj != nullptr &&
816 proj->adr_type() == TypePtr::BOTTOM) {
817 mem = proj;
818 }
819 }
820 } else {
821 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
822 Node* u = c->fast_out(i);
823 if (u->is_Proj() &&
824 u->bottom_type() == Type::MEMORY &&
825 u->adr_type() == TypePtr::BOTTOM) {
826 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
827 assert(mem == nullptr, "only one proj");
828 mem = u;
829 }
1050 if (c != ctrl ||
1051 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1052 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1053 phase->igvn().rehash_node_delayed(u);
1054 int nb = u->replace_edge(ctrl, region, &phase->igvn());
1055 if (u->is_CFG()) {
1056 if (phase->idom(u) == ctrl) {
1057 phase->set_idom(u, region, phase->dom_depth(region));
1058 }
1059 } else if (phase->get_ctrl(u) == ctrl) {
1060 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1061 uses.push(u);
1062 }
1063 assert(nb == 1, "more than 1 ctrl input?");
1064 --i, imax -= nb;
1065 }
1066 }
1067 }
1068 }
1069
1070 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections& projs, PhaseIdealLoop* phase) {
1071 Node* region = nullptr;
1072 while (c != ctrl) {
1073 if (c->is_Region()) {
1074 region = c;
1075 }
1076 c = phase->idom(c);
1077 }
1078 assert(region != nullptr, "");
1079 Node* phi = new PhiNode(region, n->bottom_type());
1080 for (uint j = 1; j < region->req(); j++) {
1081 Node* in = region->in(j);
1082 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1083 phi->init_req(j, n);
1084 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1085 phi->init_req(j, n_clone);
1086 } else {
1087 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1088 }
1089 }
1090 phase->register_new_node(phi, region);
1091 return phi;
1092 }
1093
1094 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1095 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1096
1097 Unique_Node_List uses;
1098 Node_Stack stack(0);
1099 Node_List clones;
1100 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1101 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1102
1103 Node* ctrl = phase->get_ctrl(lrb);
1104 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1166 if (phase->has_ctrl(in)) {
1167 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1168 #ifdef ASSERT
1169 for (uint i = 0; i < stack.size(); i++) {
1170 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1171 }
1172 #endif
1173 stack.push(in, 0);
1174 }
1175 } else {
1176 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1177 }
1178 }
1179 } else {
1180 phase->set_ctrl(n, call->in(0));
1181 stack.pop();
1182 }
1183 } while(stack.size() > 0);
1184 continue;
1185 }
1186 CallProjections projs;
1187 call->extract_projections(&projs, false, false);
1188
1189 #ifdef ASSERT
1190 VectorSet cloned;
1191 #endif
1192 Node* lrb_clone = lrb->clone();
1193 phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1194 phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1195
1196 stack.push(lrb, 0);
1197 clones.push(lrb_clone);
1198
1199 do {
1200 assert(stack.size() == clones.size(), "");
1201 Node* n = stack.node();
1202 #ifdef ASSERT
1203 if (n->is_Load()) {
1204 Node* mem = n->in(MemNode::Memory);
1205 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1206 Node* u = mem->fast_out(j);
1207 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1208 }
1209 }
1210 #endif
1211 uint idx = stack.index();
1212 Node* n_clone = clones.at(clones.size()-1);
1213 if (idx < n->outcnt()) {
1214 Node* u = n->raw_out(idx);
1215 Node* c = phase->ctrl_or_self(u);
1216 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) {
1217 stack.set_index(idx+1);
1218 assert(!u->is_CFG(), "");
1219 stack.push(u, 0);
1220 assert(!cloned.test_set(u->_idx), "only one clone");
1221 Node* u_clone = u->clone();
1222 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1223 assert(nb > 0, "should have replaced some uses");
1224 phase->register_new_node(u_clone, projs.catchall_catchproj);
1225 clones.push(u_clone);
1226 phase->set_ctrl(u, projs.fallthrough_catchproj);
1227 } else {
1228 bool replaced = false;
1229 if (u->is_Phi()) {
1230 for (uint k = 1; k < u->req(); k++) {
1231 if (u->in(k) == n) {
1232 if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1233 phase->igvn().replace_input_of(u, k, n_clone);
1234 replaced = true;
1235 } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1236 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1237 replaced = true;
1238 }
1239 }
1240 }
1241 } else {
1242 if (phase->is_dominator(projs.catchall_catchproj, c)) {
1243 phase->igvn().rehash_node_delayed(u);
1244 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1245 assert(nb > 0, "should have replaced some uses");
1246 replaced = true;
1247 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1248 if (u->is_If()) {
1249 // Can't break If/Bool/Cmp chain
1250 assert(n->is_Bool(), "unexpected If shape");
1251 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1252 assert(n_clone->is_Bool(), "unexpected clone");
1253 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1254 Node* bol_clone = n->clone();
1255 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1256 bol_clone->set_req(1, cmp_clone);
1257
1258 Node* nn = stack.node_at(stack.size()-3);
1259 Node* nn_clone = clones.at(clones.size()-3);
1260 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1261
1262 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1263 &phase->igvn());
1264 assert(nb > 0, "should have replaced some uses");
1265
1266 phase->register_new_node(bol_clone, u->in(0));
1267 phase->register_new_node(cmp_clone, u->in(0));
2048 } else {
2049 assert(c != c->in(0), "");
2050 c = c->in(0);
2051 }
2052 }
2053 }
2054 }
2055 } while (stack.size() > 0);
2056 assert(mem != nullptr, "should have found safepoint");
2057 } else {
2058 mem = phi_mem;
2059 }
2060 return mem;
2061 }
2062
2063 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
2064 Node* c = _phase->get_ctrl(n);
2065 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
2066 assert(c == n->in(0), "");
2067 CallNode* call = c->as_Call();
2068 CallProjections projs;
2069 call->extract_projections(&projs, true, false);
2070 if (projs.catchall_memproj != nullptr) {
2071 if (projs.fallthrough_memproj == n) {
2072 c = projs.fallthrough_catchproj;
2073 } else {
2074 assert(projs.catchall_memproj == n, "");
2075 c = projs.catchall_catchproj;
2076 }
2077 }
2078 }
2079 return c;
2080 }
2081
2082 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
2083 if (_phase->has_ctrl(n))
2084 return get_ctrl(n);
2085 else {
2086 assert (n->is_CFG(), "must be a CFG node");
2087 return n;
2088 }
2089 }
2090
2091 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2092 return m != nullptr && get_ctrl(m) == c;
2093 }
2094
2095 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|
437 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
438 "sha512_implCompressMB",
439 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
440 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
441 "encodeBlock",
442 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
443 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
444 "decodeBlock",
445 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
446 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
447 "intpoly_montgomeryMult_P256",
448 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahStore },
449 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
450 "intpoly_assign",
451 { { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad }, { -1, ShenandoahNone },
452 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
453 };
454
455 if (call->is_call_to_arraycopystub()) {
456 Node* dest = nullptr;
457 const TypeTuple* args = n->as_Call()->_tf->domain_sig();
458 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
459 if (args->field_at(i)->isa_ptr()) {
460 j++;
461 if (j == 2) {
462 dest = n->in(i);
463 break;
464 }
465 }
466 }
467 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
468 !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
469 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
470 }
471 } else if (strlen(call->_name) > 5 &&
472 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
473 if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
474 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
475 }
476 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
477 // skip
558 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
559 Op_VectorizedHashCode,
560 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
561 Op_EncodeISOArray,
562 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
563 Op_CountPositives,
564 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
565 Op_CastP2X,
566 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
567 Op_StrIndexOfChar,
568 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
569 };
570
571 const int others_len = sizeof(others) / sizeof(others[0]);
572 int i = 0;
573 for (; i < others_len; i++) {
574 if (others[i].opcode == n->Opcode()) {
575 break;
576 }
577 }
578 uint stop = n->is_Call() ? n->as_Call()->tf()->domain_sig()->cnt() : n->req();
579 if (i != others_len) {
580 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
581 for (uint j = 0; j < inputs_len; j++) {
582 int pos = others[i].inputs[j].pos;
583 if (pos == -1) {
584 break;
585 }
586 if (!verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
587 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
588 }
589 }
590 for (uint j = 1; j < stop; j++) {
591 if (n->in(j) != nullptr && n->in(j)->bottom_type()->make_ptr() &&
592 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
593 uint k = 0;
594 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
595 if (k == inputs_len) {
596 fatal("arg %d for node %s not covered", j, n->Name());
597 }
598 }
778 mem_ctrl = phase->ctrl_or_self(mem);
779 }
780 return mem;
781 }
782
783 Node* ShenandoahBarrierC2Support::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
784 Node* mem = nullptr;
785 Node* c = ctrl;
786 do {
787 if (c->is_Region()) {
788 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == nullptr; i++) {
789 Node* u = c->fast_out(i);
790 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
791 if (u->adr_type() == TypePtr::BOTTOM) {
792 mem = u;
793 }
794 }
795 }
796 } else {
797 if (c->is_Call() && c->as_Call()->adr_type() != nullptr) {
798 CallProjections* projs = c->as_Call()->extract_projections(true, false);
799 if (projs->fallthrough_memproj != nullptr) {
800 if (projs->fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
801 if (projs->catchall_memproj == nullptr) {
802 mem = projs->fallthrough_memproj;
803 } else {
804 if (phase->is_dominator(projs->fallthrough_catchproj, ctrl)) {
805 mem = projs->fallthrough_memproj;
806 } else {
807 assert(phase->is_dominator(projs->catchall_catchproj, ctrl), "one proj must dominate barrier");
808 mem = projs->catchall_memproj;
809 }
810 }
811 }
812 } else {
813 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
814 if (proj != nullptr &&
815 proj->adr_type() == TypePtr::BOTTOM) {
816 mem = proj;
817 }
818 }
819 } else {
820 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
821 Node* u = c->fast_out(i);
822 if (u->is_Proj() &&
823 u->bottom_type() == Type::MEMORY &&
824 u->adr_type() == TypePtr::BOTTOM) {
825 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
826 assert(mem == nullptr, "only one proj");
827 mem = u;
828 }
1049 if (c != ctrl ||
1050 is_dominator_same_ctrl(old_c, barrier, u, phase) ||
1051 ShenandoahBarrierSetC2::is_shenandoah_state_load(u)) {
1052 phase->igvn().rehash_node_delayed(u);
1053 int nb = u->replace_edge(ctrl, region, &phase->igvn());
1054 if (u->is_CFG()) {
1055 if (phase->idom(u) == ctrl) {
1056 phase->set_idom(u, region, phase->dom_depth(region));
1057 }
1058 } else if (phase->get_ctrl(u) == ctrl) {
1059 assert(u != init_raw_mem, "should leave input raw mem above the barrier");
1060 uses.push(u);
1061 }
1062 assert(nb == 1, "more than 1 ctrl input?");
1063 --i, imax -= nb;
1064 }
1065 }
1066 }
1067 }
1068
1069 static Node* create_phis_on_call_return(Node* ctrl, Node* c, Node* n, Node* n_clone, const CallProjections* projs, PhaseIdealLoop* phase) {
1070 Node* region = nullptr;
1071 while (c != ctrl) {
1072 if (c->is_Region()) {
1073 region = c;
1074 }
1075 c = phase->idom(c);
1076 }
1077 assert(region != nullptr, "");
1078 Node* phi = new PhiNode(region, n->bottom_type());
1079 for (uint j = 1; j < region->req(); j++) {
1080 Node* in = region->in(j);
1081 if (phase->is_dominator(projs->fallthrough_catchproj, in)) {
1082 phi->init_req(j, n);
1083 } else if (phase->is_dominator(projs->catchall_catchproj, in)) {
1084 phi->init_req(j, n_clone);
1085 } else {
1086 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1087 }
1088 }
1089 phase->register_new_node(phi, region);
1090 return phi;
1091 }
1092
1093 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1094 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1095
1096 Unique_Node_List uses;
1097 Node_Stack stack(0);
1098 Node_List clones;
1099 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1100 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1101
1102 Node* ctrl = phase->get_ctrl(lrb);
1103 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1165 if (phase->has_ctrl(in)) {
1166 if (phase->is_dominator(call, phase->get_ctrl(in))) {
1167 #ifdef ASSERT
1168 for (uint i = 0; i < stack.size(); i++) {
1169 assert(stack.node_at(i) != in, "node shouldn't have been seen yet");
1170 }
1171 #endif
1172 stack.push(in, 0);
1173 }
1174 } else {
1175 assert(phase->is_dominator(in, call->in(0)), "no dependency on the call");
1176 }
1177 }
1178 } else {
1179 phase->set_ctrl(n, call->in(0));
1180 stack.pop();
1181 }
1182 } while(stack.size() > 0);
1183 continue;
1184 }
1185 CallProjections* projs = call->extract_projections(false, false);
1186 #ifdef ASSERT
1187 VectorSet cloned;
1188 #endif
1189 Node* lrb_clone = lrb->clone();
1190 phase->register_new_node(lrb_clone, projs->catchall_catchproj);
1191 phase->set_ctrl(lrb, projs->fallthrough_catchproj);
1192
1193 stack.push(lrb, 0);
1194 clones.push(lrb_clone);
1195
1196 do {
1197 assert(stack.size() == clones.size(), "");
1198 Node* n = stack.node();
1199 #ifdef ASSERT
1200 if (n->is_Load()) {
1201 Node* mem = n->in(MemNode::Memory);
1202 for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1203 Node* u = mem->fast_out(j);
1204 assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1205 }
1206 }
1207 #endif
1208 uint idx = stack.index();
1209 Node* n_clone = clones.at(clones.size()-1);
1210 if (idx < n->outcnt()) {
1211 Node* u = n->raw_out(idx);
1212 Node* c = phase->ctrl_or_self(u);
1213 if (phase->is_dominator(call, c) && phase->is_dominator(c, projs->fallthrough_proj)) {
1214 stack.set_index(idx+1);
1215 assert(!u->is_CFG(), "");
1216 stack.push(u, 0);
1217 assert(!cloned.test_set(u->_idx), "only one clone");
1218 Node* u_clone = u->clone();
1219 int nb = u_clone->replace_edge(n, n_clone, &phase->igvn());
1220 assert(nb > 0, "should have replaced some uses");
1221 phase->register_new_node(u_clone, projs->catchall_catchproj);
1222 clones.push(u_clone);
1223 phase->set_ctrl(u, projs->fallthrough_catchproj);
1224 } else {
1225 bool replaced = false;
1226 if (u->is_Phi()) {
1227 for (uint k = 1; k < u->req(); k++) {
1228 if (u->in(k) == n) {
1229 if (phase->is_dominator(projs->catchall_catchproj, u->in(0)->in(k))) {
1230 phase->igvn().replace_input_of(u, k, n_clone);
1231 replaced = true;
1232 } else if (!phase->is_dominator(projs->fallthrough_catchproj, u->in(0)->in(k))) {
1233 phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1234 replaced = true;
1235 }
1236 }
1237 }
1238 } else {
1239 if (phase->is_dominator(projs->catchall_catchproj, c)) {
1240 phase->igvn().rehash_node_delayed(u);
1241 int nb = u->replace_edge(n, n_clone, &phase->igvn());
1242 assert(nb > 0, "should have replaced some uses");
1243 replaced = true;
1244 } else if (!phase->is_dominator(projs->fallthrough_catchproj, c)) {
1245 if (u->is_If()) {
1246 // Can't break If/Bool/Cmp chain
1247 assert(n->is_Bool(), "unexpected If shape");
1248 assert(stack.node_at(stack.size()-2)->is_Cmp(), "unexpected If shape");
1249 assert(n_clone->is_Bool(), "unexpected clone");
1250 assert(clones.at(clones.size()-2)->is_Cmp(), "unexpected clone");
1251 Node* bol_clone = n->clone();
1252 Node* cmp_clone = stack.node_at(stack.size()-2)->clone();
1253 bol_clone->set_req(1, cmp_clone);
1254
1255 Node* nn = stack.node_at(stack.size()-3);
1256 Node* nn_clone = clones.at(clones.size()-3);
1257 assert(nn->Opcode() == nn_clone->Opcode(), "mismatch");
1258
1259 int nb = cmp_clone->replace_edge(nn, create_phis_on_call_return(ctrl, c, nn, nn_clone, projs, phase),
1260 &phase->igvn());
1261 assert(nb > 0, "should have replaced some uses");
1262
1263 phase->register_new_node(bol_clone, u->in(0));
1264 phase->register_new_node(cmp_clone, u->in(0));
2045 } else {
2046 assert(c != c->in(0), "");
2047 c = c->in(0);
2048 }
2049 }
2050 }
2051 }
2052 } while (stack.size() > 0);
2053 assert(mem != nullptr, "should have found safepoint");
2054 } else {
2055 mem = phi_mem;
2056 }
2057 return mem;
2058 }
2059
2060 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
2061 Node* c = _phase->get_ctrl(n);
2062 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Call()) {
2063 assert(c == n->in(0), "");
2064 CallNode* call = c->as_Call();
2065 CallProjections* projs = call->extract_projections(true, false);
2066 if (projs->catchall_memproj != nullptr) {
2067 if (projs->fallthrough_memproj == n) {
2068 c = projs->fallthrough_catchproj;
2069 } else {
2070 assert(projs->catchall_memproj == n, "");
2071 c = projs->catchall_catchproj;
2072 }
2073 }
2074 }
2075 return c;
2076 }
2077
2078 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
2079 if (_phase->has_ctrl(n))
2080 return get_ctrl(n);
2081 else {
2082 assert (n->is_CFG(), "must be a CFG node");
2083 return n;
2084 }
2085 }
2086
2087 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
2088 return m != nullptr && get_ctrl(m) == c;
2089 }
2090
2091 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|