948
949 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
950 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
951 region->init_req(_heap_unstable, ctrl);
952 mem_phi->init_req(_heap_unstable, mem);
953
954 // Wire up the actual arraycopy stub now
955 ctrl = phase->transform_later(region);
956 mem = phase->transform_later(mem_phi);
957
958 const char* name = "arraycopy";
959 call = phase->make_leaf_call(ctrl, mem,
960 OptoRuntime::fast_arraycopy_Type(),
961 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
962 name, TypeRawPtr::BOTTOM,
963 src, dest, length
964 LP64_ONLY(COMMA phase->top()));
965 call = phase->transform_later(call);
966
967 // Hook up the whole thing into the graph
968 phase->igvn().replace_node(ac, call);
969 } else {
970 BarrierSetC2::clone_at_expansion(phase, ac);
971 }
972 }
973
974
975 // Support for macro expanded GC barriers
976 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
977 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
978 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
979 }
980 }
981
982 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
983 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
984 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
985 }
986 }
987
988 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
989 if (is_shenandoah_wb_pre_call(node)) {
990 shenandoah_eliminate_wb_pre(node, ¯o->igvn());
991 }
992 if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
993 for (DUIterator_Last imin, i = node->last_outs(imin); i >= imin; --i) {
994 Node* shift = node->last_out(i);
995 for (DUIterator_Last kmin, k = shift->last_outs(kmin); k >= kmin; --k) {
996 Node* addp = shift->last_out(k);
997 for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
998 Node* mem = addp->last_out(j);
999 if (UseCondCardMark && mem->is_Load()) {
1000 assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
1001 // The load is checking if the card has been written so
1002 // replace it with zero to fold the test.
1003 macro->replace_node(mem, macro->intcon(0));
1004 continue;
1005 }
1006 assert(mem->is_Store(), "store required");
1007 macro->replace_node(mem, mem->in(MemNode::Memory));
1008 }
1009 }
1010 }
1011 }
1012 }
1013
1014 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
1015 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
1016 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
1017 c = c->unique_ctrl_out();
1018 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1019 c = c->unique_ctrl_out();
1020 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1021 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1022 assert(iff->is_If(), "expect test");
1023 if (!is_shenandoah_marking_if(igvn, iff)) {
1024 c = c->unique_ctrl_out();
1025 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1026 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1027 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
1121 if (if_ctrl != load_ctrl) {
1122 // Skip possible CProj->NeverBranch in infinite loops
1123 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1124 && if_ctrl->in(0)->is_NeverBranch()) {
1125 if_ctrl = if_ctrl->in(0)->in(0);
1126 }
1127 }
1128 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
1129 }
1130 }
1131 }
1132 }
1133 }
1134 }
1135 }
1136 }
1137 #endif
1138
1139 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1140 if (is_shenandoah_wb_pre_call(n)) {
1141 uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain()->cnt();
1142 if (n->req() > cnt) {
1143 Node* addp = n->in(cnt);
1144 if (has_only_shenandoah_wb_pre_uses(addp)) {
1145 n->del_req(cnt);
1146 if (can_reshape) {
1147 phase->is_IterGVN()->_worklist.push(addp);
1148 }
1149 return n;
1150 }
1151 }
1152 }
1153 if (n->Opcode() == Op_CmpP) {
1154 Node* in1 = n->in(1);
1155 Node* in2 = n->in(2);
1156
1157 // If one input is null, then step over the strong LRB barriers on the other input
1158 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1159 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1160 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1161 in2 = step_over_gc_barrier(in2);
1207 return nullptr;
1208 }
1209
1210 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1211 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1212 Node* u = n->fast_out(i);
1213 if (!is_shenandoah_wb_pre_call(u)) {
1214 return false;
1215 }
1216 }
1217 return n->outcnt() > 0;
1218 }
1219
1220 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1221 switch (opcode) {
1222 case Op_CallLeaf:
1223 case Op_CallLeafNoFP: {
1224 assert (n->is_Call(), "");
1225 CallNode *call = n->as_Call();
1226 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1227 uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain()->cnt();
1228 if (call->req() > cnt) {
1229 assert(call->req() == cnt + 1, "only one extra input");
1230 Node *addp = call->in(cnt);
1231 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1232 call->del_req(cnt);
1233 }
1234 }
1235 return false;
1236 }
1237 case Op_ShenandoahCompareAndSwapP:
1238 case Op_ShenandoahCompareAndSwapN:
1239 case Op_ShenandoahWeakCompareAndSwapN:
1240 case Op_ShenandoahWeakCompareAndSwapP:
1241 case Op_ShenandoahCompareAndExchangeP:
1242 case Op_ShenandoahCompareAndExchangeN:
1243 return true;
1244 case Op_ShenandoahLoadReferenceBarrier:
1245 assert(false, "should have been expanded already");
1246 return true;
1247 default:
|
948
949 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
950 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
951 region->init_req(_heap_unstable, ctrl);
952 mem_phi->init_req(_heap_unstable, mem);
953
954 // Wire up the actual arraycopy stub now
955 ctrl = phase->transform_later(region);
956 mem = phase->transform_later(mem_phi);
957
958 const char* name = "arraycopy";
959 call = phase->make_leaf_call(ctrl, mem,
960 OptoRuntime::fast_arraycopy_Type(),
961 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
962 name, TypeRawPtr::BOTTOM,
963 src, dest, length
964 LP64_ONLY(COMMA phase->top()));
965 call = phase->transform_later(call);
966
967 // Hook up the whole thing into the graph
968 phase->replace_node(ac, call);
969 } else {
970 BarrierSetC2::clone_at_expansion(phase, ac);
971 }
972 }
973
974
975 // Support for macro expanded GC barriers
976 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
977 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
978 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
979 }
980 }
981
982 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
983 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
984 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
985 }
986 }
987
988 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
989 if (is_shenandoah_wb_pre_call(node)) {
990 shenandoah_eliminate_wb_pre(node, igvn);
991 }
992 if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
993 for (DUIterator_Last imin, i = node->last_outs(imin); i >= imin; --i) {
994 Node* shift = node->last_out(i);
995 for (DUIterator_Last kmin, k = shift->last_outs(kmin); k >= kmin; --k) {
996 Node* addp = shift->last_out(k);
997 for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
998 Node* mem = addp->last_out(j);
999 if (UseCondCardMark && mem->is_Load()) {
1000 assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
1001 // The load is checking if the card has been written so
1002 // replace it with zero to fold the test.
1003 igvn->replace_node(mem, igvn->intcon(0));
1004 continue;
1005 }
1006 assert(mem->is_Store(), "store required");
1007 igvn->replace_node(mem, mem->in(MemNode::Memory));
1008 }
1009 }
1010 }
1011 }
1012 }
1013
1014 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
1015 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
1016 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
1017 c = c->unique_ctrl_out();
1018 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1019 c = c->unique_ctrl_out();
1020 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1021 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1022 assert(iff->is_If(), "expect test");
1023 if (!is_shenandoah_marking_if(igvn, iff)) {
1024 c = c->unique_ctrl_out();
1025 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1026 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1027 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
1121 if (if_ctrl != load_ctrl) {
1122 // Skip possible CProj->NeverBranch in infinite loops
1123 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1124 && if_ctrl->in(0)->is_NeverBranch()) {
1125 if_ctrl = if_ctrl->in(0)->in(0);
1126 }
1127 }
1128 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
1129 }
1130 }
1131 }
1132 }
1133 }
1134 }
1135 }
1136 }
1137 #endif
1138
1139 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1140 if (is_shenandoah_wb_pre_call(n)) {
1141 uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain_sig()->cnt();
1142 if (n->req() > cnt) {
1143 Node* addp = n->in(cnt);
1144 if (has_only_shenandoah_wb_pre_uses(addp)) {
1145 n->del_req(cnt);
1146 if (can_reshape) {
1147 phase->is_IterGVN()->_worklist.push(addp);
1148 }
1149 return n;
1150 }
1151 }
1152 }
1153 if (n->Opcode() == Op_CmpP) {
1154 Node* in1 = n->in(1);
1155 Node* in2 = n->in(2);
1156
1157 // If one input is null, then step over the strong LRB barriers on the other input
1158 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1159 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1160 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1161 in2 = step_over_gc_barrier(in2);
1207 return nullptr;
1208 }
1209
1210 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1211 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1212 Node* u = n->fast_out(i);
1213 if (!is_shenandoah_wb_pre_call(u)) {
1214 return false;
1215 }
1216 }
1217 return n->outcnt() > 0;
1218 }
1219
1220 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1221 switch (opcode) {
1222 case Op_CallLeaf:
1223 case Op_CallLeafNoFP: {
1224 assert (n->is_Call(), "");
1225 CallNode *call = n->as_Call();
1226 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1227 uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain_sig()->cnt();
1228 if (call->req() > cnt) {
1229 assert(call->req() == cnt + 1, "only one extra input");
1230 Node *addp = call->in(cnt);
1231 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1232 call->del_req(cnt);
1233 }
1234 }
1235 return false;
1236 }
1237 case Op_ShenandoahCompareAndSwapP:
1238 case Op_ShenandoahCompareAndSwapN:
1239 case Op_ShenandoahWeakCompareAndSwapN:
1240 case Op_ShenandoahWeakCompareAndSwapP:
1241 case Op_ShenandoahCompareAndExchangeP:
1242 case Op_ShenandoahCompareAndExchangeN:
1243 return true;
1244 case Op_ShenandoahLoadReferenceBarrier:
1245 assert(false, "should have been expanded already");
1246 return true;
1247 default:
|