879 Node* ctrl = ac->in(TypeFunc::Control);
880 Node* mem = ac->in(TypeFunc::Memory);
881 Node* src_base = ac->in(ArrayCopyNode::Src);
882 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
883 Node* dest_base = ac->in(ArrayCopyNode::Dest);
884 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
885 Node* length = ac->in(ArrayCopyNode::Length);
886
887 Node* src = phase->basic_plus_adr(src_base, src_offset);
888 Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
889
890 if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
891 // Check if heap is has forwarded objects. If it does, we need to call into the special
892 // routine that would fix up source references before we can continue.
893
894 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
895 Node* region = new RegionNode(PATH_LIMIT);
896 Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
897
898 Node* thread = phase->transform_later(new ThreadLocalNode());
899 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
900 Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
901
902 uint gc_state_idx = Compile::AliasIdxRaw;
903 const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
904 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
905
906 Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
907 Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
908 Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
909 Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
910
911 IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
912 Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
913 Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
914
915 // Heap is stable, no need to do anything additional
916 region->init_req(_heap_stable, stable_ctrl);
917 mem_phi->init_req(_heap_stable, mem);
918
919 // Heap is unstable, call into clone barrier stub
927
928 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
929 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
930 region->init_req(_heap_unstable, ctrl);
931 mem_phi->init_req(_heap_unstable, mem);
932
933 // Wire up the actual arraycopy stub now
934 ctrl = phase->transform_later(region);
935 mem = phase->transform_later(mem_phi);
936
937 const char* name = "arraycopy";
938 call = phase->make_leaf_call(ctrl, mem,
939 OptoRuntime::fast_arraycopy_Type(),
940 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
941 name, TypeRawPtr::BOTTOM,
942 src, dest, length
943 LP64_ONLY(COMMA phase->top()));
944 call = phase->transform_later(call);
945
946 // Hook up the whole thing into the graph
947 phase->igvn().replace_node(ac, call);
948 } else {
949 BarrierSetC2::clone_at_expansion(phase, ac);
950 }
951 }
952
953
954 // Support for macro expanded GC barriers
955 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
956 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
957 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
958 }
959 }
960
961 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
962 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
963 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
964 }
965 }
966
967 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
968 if (is_shenandoah_wb_pre_call(node)) {
969 shenandoah_eliminate_wb_pre(node, ¯o->igvn());
970 }
971 if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
972 Node* shift = node->unique_out();
973 Node* addp = shift->unique_out();
974 for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
975 Node* mem = addp->last_out(j);
976 if (UseCondCardMark && mem->is_Load()) {
977 assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
978 // The load is checking if the card has been written so
979 // replace it with zero to fold the test.
980 macro->replace_node(mem, macro->intcon(0));
981 continue;
982 }
983 assert(mem->is_Store(), "store required");
984 macro->replace_node(mem, mem->in(MemNode::Memory));
985 }
986 }
987 }
988
989 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
990 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
991 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
992 c = c->unique_ctrl_out();
993 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
994 c = c->unique_ctrl_out();
995 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
996 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
997 assert(iff->is_If(), "expect test");
998 if (!is_shenandoah_marking_if(igvn, iff)) {
999 c = c->unique_ctrl_out();
1000 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1001 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1002 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
1003 }
1004 Node* cmpx = iff->in(1)->in(1);
1096 if (if_ctrl != load_ctrl) {
1097 // Skip possible CProj->NeverBranch in infinite loops
1098 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1099 && if_ctrl->in(0)->is_NeverBranch()) {
1100 if_ctrl = if_ctrl->in(0)->in(0);
1101 }
1102 }
1103 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
1104 }
1105 }
1106 }
1107 }
1108 }
1109 }
1110 }
1111 }
1112 #endif
1113
1114 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1115 if (is_shenandoah_wb_pre_call(n)) {
1116 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_Type()->domain()->cnt();
1117 if (n->req() > cnt) {
1118 Node* addp = n->in(cnt);
1119 if (has_only_shenandoah_wb_pre_uses(addp)) {
1120 n->del_req(cnt);
1121 if (can_reshape) {
1122 phase->is_IterGVN()->_worklist.push(addp);
1123 }
1124 return n;
1125 }
1126 }
1127 }
1128 if (n->Opcode() == Op_CmpP) {
1129 Node* in1 = n->in(1);
1130 Node* in2 = n->in(2);
1131
1132 // If one input is null, then step over the strong LRB barriers on the other input
1133 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1134 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1135 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1136 in2 = step_over_gc_barrier(in2);
1182 return nullptr;
1183 }
1184
1185 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1186 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1187 Node* u = n->fast_out(i);
1188 if (!is_shenandoah_wb_pre_call(u)) {
1189 return false;
1190 }
1191 }
1192 return n->outcnt() > 0;
1193 }
1194
1195 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1196 switch (opcode) {
1197 case Op_CallLeaf:
1198 case Op_CallLeafNoFP: {
1199 assert (n->is_Call(), "");
1200 CallNode *call = n->as_Call();
1201 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1202 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_Type()->domain()->cnt();
1203 if (call->req() > cnt) {
1204 assert(call->req() == cnt + 1, "only one extra input");
1205 Node *addp = call->in(cnt);
1206 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1207 call->del_req(cnt);
1208 }
1209 }
1210 return false;
1211 }
1212 case Op_ShenandoahCompareAndSwapP:
1213 case Op_ShenandoahCompareAndSwapN:
1214 case Op_ShenandoahWeakCompareAndSwapN:
1215 case Op_ShenandoahWeakCompareAndSwapP:
1216 case Op_ShenandoahCompareAndExchangeP:
1217 case Op_ShenandoahCompareAndExchangeN:
1218 return true;
1219 case Op_ShenandoahLoadReferenceBarrier:
1220 assert(false, "should have been expanded already");
1221 return true;
1222 default:
|
879 Node* ctrl = ac->in(TypeFunc::Control);
880 Node* mem = ac->in(TypeFunc::Memory);
881 Node* src_base = ac->in(ArrayCopyNode::Src);
882 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
883 Node* dest_base = ac->in(ArrayCopyNode::Dest);
884 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
885 Node* length = ac->in(ArrayCopyNode::Length);
886
887 Node* src = phase->basic_plus_adr(src_base, src_offset);
888 Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
889
890 if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
891 // Check if heap is has forwarded objects. If it does, we need to call into the special
892 // routine that would fix up source references before we can continue.
893
894 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
895 Node* region = new RegionNode(PATH_LIMIT);
896 Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
897
898 Node* thread = phase->transform_later(new ThreadLocalNode());
899 Node* offset = phase->MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
900 Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
901
902 uint gc_state_idx = Compile::AliasIdxRaw;
903 const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
904 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
905
906 Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
907 Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
908 Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
909 Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
910
911 IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
912 Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
913 Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
914
915 // Heap is stable, no need to do anything additional
916 region->init_req(_heap_stable, stable_ctrl);
917 mem_phi->init_req(_heap_stable, mem);
918
919 // Heap is unstable, call into clone barrier stub
927
928 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
929 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
930 region->init_req(_heap_unstable, ctrl);
931 mem_phi->init_req(_heap_unstable, mem);
932
933 // Wire up the actual arraycopy stub now
934 ctrl = phase->transform_later(region);
935 mem = phase->transform_later(mem_phi);
936
937 const char* name = "arraycopy";
938 call = phase->make_leaf_call(ctrl, mem,
939 OptoRuntime::fast_arraycopy_Type(),
940 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
941 name, TypeRawPtr::BOTTOM,
942 src, dest, length
943 LP64_ONLY(COMMA phase->top()));
944 call = phase->transform_later(call);
945
946 // Hook up the whole thing into the graph
947 phase->replace_node(ac, call);
948 } else {
949 BarrierSetC2::clone_at_expansion(phase, ac);
950 }
951 }
952
953
954 // Support for macro expanded GC barriers
955 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
956 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
957 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
958 }
959 }
960
961 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
962 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
963 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
964 }
965 }
966
967 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
968 if (is_shenandoah_wb_pre_call(node)) {
969 shenandoah_eliminate_wb_pre(node, igvn);
970 }
971 if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
972 Node* shift = node->unique_out();
973 Node* addp = shift->unique_out();
974 for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
975 Node* mem = addp->last_out(j);
976 if (UseCondCardMark && mem->is_Load()) {
977 assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
978 // The load is checking if the card has been written so
979 // replace it with zero to fold the test.
980 igvn->replace_node(mem, igvn->intcon(0));
981 continue;
982 }
983 assert(mem->is_Store(), "store required");
984 igvn->replace_node(mem, mem->in(MemNode::Memory));
985 }
986 }
987 }
988
989 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
990 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
991 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
992 c = c->unique_ctrl_out();
993 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
994 c = c->unique_ctrl_out();
995 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
996 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
997 assert(iff->is_If(), "expect test");
998 if (!is_shenandoah_marking_if(igvn, iff)) {
999 c = c->unique_ctrl_out();
1000 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1001 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1002 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
1003 }
1004 Node* cmpx = iff->in(1)->in(1);
1096 if (if_ctrl != load_ctrl) {
1097 // Skip possible CProj->NeverBranch in infinite loops
1098 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1099 && if_ctrl->in(0)->is_NeverBranch()) {
1100 if_ctrl = if_ctrl->in(0)->in(0);
1101 }
1102 }
1103 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
1104 }
1105 }
1106 }
1107 }
1108 }
1109 }
1110 }
1111 }
1112 #endif
1113
1114 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1115 if (is_shenandoah_wb_pre_call(n)) {
1116 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_Type()->domain_sig()->cnt();
1117 if (n->req() > cnt) {
1118 Node* addp = n->in(cnt);
1119 if (has_only_shenandoah_wb_pre_uses(addp)) {
1120 n->del_req(cnt);
1121 if (can_reshape) {
1122 phase->is_IterGVN()->_worklist.push(addp);
1123 }
1124 return n;
1125 }
1126 }
1127 }
1128 if (n->Opcode() == Op_CmpP) {
1129 Node* in1 = n->in(1);
1130 Node* in2 = n->in(2);
1131
1132 // If one input is null, then step over the strong LRB barriers on the other input
1133 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1134 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1135 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1136 in2 = step_over_gc_barrier(in2);
1182 return nullptr;
1183 }
1184
1185 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1186 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1187 Node* u = n->fast_out(i);
1188 if (!is_shenandoah_wb_pre_call(u)) {
1189 return false;
1190 }
1191 }
1192 return n->outcnt() > 0;
1193 }
1194
1195 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1196 switch (opcode) {
1197 case Op_CallLeaf:
1198 case Op_CallLeafNoFP: {
1199 assert (n->is_Call(), "");
1200 CallNode *call = n->as_Call();
1201 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1202 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_Type()->domain_sig()->cnt();
1203 if (call->req() > cnt) {
1204 assert(call->req() == cnt + 1, "only one extra input");
1205 Node *addp = call->in(cnt);
1206 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1207 call->del_req(cnt);
1208 }
1209 }
1210 return false;
1211 }
1212 case Op_ShenandoahCompareAndSwapP:
1213 case Op_ShenandoahCompareAndSwapN:
1214 case Op_ShenandoahWeakCompareAndSwapN:
1215 case Op_ShenandoahWeakCompareAndSwapP:
1216 case Op_ShenandoahCompareAndExchangeP:
1217 case Op_ShenandoahCompareAndExchangeN:
1218 return true;
1219 case Op_ShenandoahLoadReferenceBarrier:
1220 assert(false, "should have been expanded already");
1221 return true;
1222 default:
|