948
949 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
950 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
951 region->init_req(_heap_unstable, ctrl);
952 mem_phi->init_req(_heap_unstable, mem);
953
954 // Wire up the actual arraycopy stub now
955 ctrl = phase->transform_later(region);
956 mem = phase->transform_later(mem_phi);
957
958 const char* name = "arraycopy";
959 call = phase->make_leaf_call(ctrl, mem,
960 OptoRuntime::fast_arraycopy_Type(),
961 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
962 name, TypeRawPtr::BOTTOM,
963 src, dest, length
964 LP64_ONLY(COMMA phase->top()));
965 call = phase->transform_later(call);
966
967 // Hook up the whole thing into the graph
968 phase->igvn().replace_node(ac, call);
969 } else {
970 BarrierSetC2::clone_at_expansion(phase, ac);
971 }
972 }
973
974
975 // Support for macro expanded GC barriers
976 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
977 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
978 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
979 }
980 }
981
982 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
983 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
984 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
985 }
986 }
987
988 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
989 if (is_shenandoah_wb_pre_call(node)) {
990 shenandoah_eliminate_wb_pre(node, ¯o->igvn());
991 }
992 if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
993 Node* shift = node->unique_out();
994 Node* addp = shift->unique_out();
995 for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
996 Node* mem = addp->last_out(j);
997 if (UseCondCardMark && mem->is_Load()) {
998 assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
999 // The load is checking if the card has been written so
1000 // replace it with zero to fold the test.
1001 macro->replace_node(mem, macro->intcon(0));
1002 continue;
1003 }
1004 assert(mem->is_Store(), "store required");
1005 macro->replace_node(mem, mem->in(MemNode::Memory));
1006 }
1007 }
1008 }
1009
1010 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
1011 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
1012 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
1013 c = c->unique_ctrl_out();
1014 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1015 c = c->unique_ctrl_out();
1016 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1017 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1018 assert(iff->is_If(), "expect test");
1019 if (!is_shenandoah_marking_if(igvn, iff)) {
1020 c = c->unique_ctrl_out();
1021 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1022 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1023 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
1024 }
1025 Node* cmpx = iff->in(1)->in(1);
1117 if (if_ctrl != load_ctrl) {
1118 // Skip possible CProj->NeverBranch in infinite loops
1119 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1120 && if_ctrl->in(0)->is_NeverBranch()) {
1121 if_ctrl = if_ctrl->in(0)->in(0);
1122 }
1123 }
1124 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
1125 }
1126 }
1127 }
1128 }
1129 }
1130 }
1131 }
1132 }
1133 #endif
1134
1135 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1136 if (is_shenandoah_wb_pre_call(n)) {
1137 uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain()->cnt();
1138 if (n->req() > cnt) {
1139 Node* addp = n->in(cnt);
1140 if (has_only_shenandoah_wb_pre_uses(addp)) {
1141 n->del_req(cnt);
1142 if (can_reshape) {
1143 phase->is_IterGVN()->_worklist.push(addp);
1144 }
1145 return n;
1146 }
1147 }
1148 }
1149 if (n->Opcode() == Op_CmpP) {
1150 Node* in1 = n->in(1);
1151 Node* in2 = n->in(2);
1152
1153 // If one input is null, then step over the strong LRB barriers on the other input
1154 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1155 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1156 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1157 in2 = step_over_gc_barrier(in2);
1203 return nullptr;
1204 }
1205
1206 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1207 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1208 Node* u = n->fast_out(i);
1209 if (!is_shenandoah_wb_pre_call(u)) {
1210 return false;
1211 }
1212 }
1213 return n->outcnt() > 0;
1214 }
1215
1216 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1217 switch (opcode) {
1218 case Op_CallLeaf:
1219 case Op_CallLeafNoFP: {
1220 assert (n->is_Call(), "");
1221 CallNode *call = n->as_Call();
1222 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1223 uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain()->cnt();
1224 if (call->req() > cnt) {
1225 assert(call->req() == cnt + 1, "only one extra input");
1226 Node *addp = call->in(cnt);
1227 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1228 call->del_req(cnt);
1229 }
1230 }
1231 return false;
1232 }
1233 case Op_ShenandoahCompareAndSwapP:
1234 case Op_ShenandoahCompareAndSwapN:
1235 case Op_ShenandoahWeakCompareAndSwapN:
1236 case Op_ShenandoahWeakCompareAndSwapP:
1237 case Op_ShenandoahCompareAndExchangeP:
1238 case Op_ShenandoahCompareAndExchangeN:
1239 return true;
1240 case Op_ShenandoahLoadReferenceBarrier:
1241 assert(false, "should have been expanded already");
1242 return true;
1243 default:
|
948
949 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
950 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
951 region->init_req(_heap_unstable, ctrl);
952 mem_phi->init_req(_heap_unstable, mem);
953
954 // Wire up the actual arraycopy stub now
955 ctrl = phase->transform_later(region);
956 mem = phase->transform_later(mem_phi);
957
958 const char* name = "arraycopy";
959 call = phase->make_leaf_call(ctrl, mem,
960 OptoRuntime::fast_arraycopy_Type(),
961 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
962 name, TypeRawPtr::BOTTOM,
963 src, dest, length
964 LP64_ONLY(COMMA phase->top()));
965 call = phase->transform_later(call);
966
967 // Hook up the whole thing into the graph
968 phase->replace_node(ac, call);
969 } else {
970 BarrierSetC2::clone_at_expansion(phase, ac);
971 }
972 }
973
974
975 // Support for macro expanded GC barriers
976 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
977 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
978 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
979 }
980 }
981
982 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
983 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
984 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
985 }
986 }
987
988 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
989 if (is_shenandoah_wb_pre_call(node)) {
990 shenandoah_eliminate_wb_pre(node, igvn);
991 }
992 if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
993 Node* shift = node->unique_out();
994 Node* addp = shift->unique_out();
995 for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
996 Node* mem = addp->last_out(j);
997 if (UseCondCardMark && mem->is_Load()) {
998 assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
999 // The load is checking if the card has been written so
1000 // replace it with zero to fold the test.
1001 igvn->replace_node(mem, igvn->intcon(0));
1002 continue;
1003 }
1004 assert(mem->is_Store(), "store required");
1005 igvn->replace_node(mem, mem->in(MemNode::Memory));
1006 }
1007 }
1008 }
1009
1010 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
1011 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
1012 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
1013 c = c->unique_ctrl_out();
1014 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1015 c = c->unique_ctrl_out();
1016 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1017 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1018 assert(iff->is_If(), "expect test");
1019 if (!is_shenandoah_marking_if(igvn, iff)) {
1020 c = c->unique_ctrl_out();
1021 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1022 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1023 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
1024 }
1025 Node* cmpx = iff->in(1)->in(1);
1117 if (if_ctrl != load_ctrl) {
1118 // Skip possible CProj->NeverBranch in infinite loops
1119 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1120 && if_ctrl->in(0)->is_NeverBranch()) {
1121 if_ctrl = if_ctrl->in(0)->in(0);
1122 }
1123 }
1124 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
1125 }
1126 }
1127 }
1128 }
1129 }
1130 }
1131 }
1132 }
1133 #endif
1134
1135 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1136 if (is_shenandoah_wb_pre_call(n)) {
1137 uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain_sig()->cnt();
1138 if (n->req() > cnt) {
1139 Node* addp = n->in(cnt);
1140 if (has_only_shenandoah_wb_pre_uses(addp)) {
1141 n->del_req(cnt);
1142 if (can_reshape) {
1143 phase->is_IterGVN()->_worklist.push(addp);
1144 }
1145 return n;
1146 }
1147 }
1148 }
1149 if (n->Opcode() == Op_CmpP) {
1150 Node* in1 = n->in(1);
1151 Node* in2 = n->in(2);
1152
1153 // If one input is null, then step over the strong LRB barriers on the other input
1154 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1155 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1156 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1157 in2 = step_over_gc_barrier(in2);
1203 return nullptr;
1204 }
1205
1206 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1207 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1208 Node* u = n->fast_out(i);
1209 if (!is_shenandoah_wb_pre_call(u)) {
1210 return false;
1211 }
1212 }
1213 return n->outcnt() > 0;
1214 }
1215
1216 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1217 switch (opcode) {
1218 case Op_CallLeaf:
1219 case Op_CallLeafNoFP: {
1220 assert (n->is_Call(), "");
1221 CallNode *call = n->as_Call();
1222 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1223 uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain_sig()->cnt();
1224 if (call->req() > cnt) {
1225 assert(call->req() == cnt + 1, "only one extra input");
1226 Node *addp = call->in(cnt);
1227 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1228 call->del_req(cnt);
1229 }
1230 }
1231 return false;
1232 }
1233 case Op_ShenandoahCompareAndSwapP:
1234 case Op_ShenandoahCompareAndSwapN:
1235 case Op_ShenandoahWeakCompareAndSwapN:
1236 case Op_ShenandoahWeakCompareAndSwapP:
1237 case Op_ShenandoahCompareAndExchangeP:
1238 case Op_ShenandoahCompareAndExchangeN:
1239 return true;
1240 case Op_ShenandoahLoadReferenceBarrier:
1241 assert(false, "should have been expanded already");
1242 return true;
1243 default:
|