872 Node* ctrl = ac->in(TypeFunc::Control);
873 Node* mem = ac->in(TypeFunc::Memory);
874 Node* src_base = ac->in(ArrayCopyNode::Src);
875 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
876 Node* dest_base = ac->in(ArrayCopyNode::Dest);
877 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
878 Node* length = ac->in(ArrayCopyNode::Length);
879
880 Node* src = phase->basic_plus_adr(src_base, src_offset);
881 Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
882
883 if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
884 // Check if heap is has forwarded objects. If it does, we need to call into the special
885 // routine that would fix up source references before we can continue.
886
887 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
888 Node* region = new RegionNode(PATH_LIMIT);
889 Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
890
891 Node* thread = phase->transform_later(new ThreadLocalNode());
892 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
893 Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
894
895 uint gc_state_idx = Compile::AliasIdxRaw;
896 const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
897 DEBUG_ONLY(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
898
899 Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
900 Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
901 Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
902 Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
903
904 IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
905 Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
906 Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
907
908 // Heap is stable, no need to do anything additional
909 region->init_req(_heap_stable, stable_ctrl);
910 mem_phi->init_req(_heap_stable, mem);
911
912 // Heap is unstable, call into clone barrier stub
920
921 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
922 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
923 region->init_req(_heap_unstable, ctrl);
924 mem_phi->init_req(_heap_unstable, mem);
925
926 // Wire up the actual arraycopy stub now
927 ctrl = phase->transform_later(region);
928 mem = phase->transform_later(mem_phi);
929
930 const char* name = "arraycopy";
931 call = phase->make_leaf_call(ctrl, mem,
932 OptoRuntime::fast_arraycopy_Type(),
933 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
934 name, TypeRawPtr::BOTTOM,
935 src, dest, length
936 LP64_ONLY(COMMA phase->top()));
937 call = phase->transform_later(call);
938
939 // Hook up the whole thing into the graph
940 phase->igvn().replace_node(ac, call);
941 } else {
942 BarrierSetC2::clone_at_expansion(phase, ac);
943 }
944 }
945
946
947 // Support for macro expanded GC barriers
948 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
949 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
950 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
951 }
952 }
953
954 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
955 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
956 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
957 }
958 }
959
960 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
961 if (is_shenandoah_wb_pre_call(node)) {
962 shenandoah_eliminate_wb_pre(node, ¯o->igvn());
963 }
964 if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
965 Node* shift = node->unique_out();
966 Node* addp = shift->unique_out();
967 for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
968 Node* mem = addp->last_out(j);
969 if (UseCondCardMark && mem->is_Load()) {
970 assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
971 // The load is checking if the card has been written so
972 // replace it with zero to fold the test.
973 macro->replace_node(mem, macro->intcon(0));
974 continue;
975 }
976 assert(mem->is_Store(), "store required");
977 macro->replace_node(mem, mem->in(MemNode::Memory));
978 }
979 }
980 }
981
982 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
983 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
984 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
985 c = c->unique_ctrl_out();
986 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
987 c = c->unique_ctrl_out();
988 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
989 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
990 assert(iff->is_If(), "expect test");
991 if (!is_shenandoah_marking_if(igvn, iff)) {
992 c = c->unique_ctrl_out();
993 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
994 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
995 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
996 }
997 Node* cmpx = iff->in(1)->in(1);
1089 if (if_ctrl != load_ctrl) {
1090 // Skip possible CProj->NeverBranch in infinite loops
1091 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1092 && if_ctrl->in(0)->is_NeverBranch()) {
1093 if_ctrl = if_ctrl->in(0)->in(0);
1094 }
1095 }
1096 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
1097 }
1098 }
1099 }
1100 }
1101 }
1102 }
1103 }
1104 }
1105 #endif
1106
1107 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1108 if (is_shenandoah_wb_pre_call(n)) {
1109 uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain()->cnt();
1110 if (n->req() > cnt) {
1111 Node* addp = n->in(cnt);
1112 if (has_only_shenandoah_wb_pre_uses(addp)) {
1113 n->del_req(cnt);
1114 if (can_reshape) {
1115 phase->is_IterGVN()->_worklist.push(addp);
1116 }
1117 return n;
1118 }
1119 }
1120 }
1121 if (n->Opcode() == Op_CmpP) {
1122 Node* in1 = n->in(1);
1123 Node* in2 = n->in(2);
1124
1125 // If one input is null, then step over the strong LRB barriers on the other input
1126 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1127 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1128 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1129 in2 = step_over_gc_barrier(in2);
1175 return nullptr;
1176 }
1177
1178 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1179 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1180 Node* u = n->fast_out(i);
1181 if (!is_shenandoah_wb_pre_call(u)) {
1182 return false;
1183 }
1184 }
1185 return n->outcnt() > 0;
1186 }
1187
1188 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1189 switch (opcode) {
1190 case Op_CallLeaf:
1191 case Op_CallLeafNoFP: {
1192 assert (n->is_Call(), "");
1193 CallNode *call = n->as_Call();
1194 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1195 uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain()->cnt();
1196 if (call->req() > cnt) {
1197 assert(call->req() == cnt + 1, "only one extra input");
1198 Node *addp = call->in(cnt);
1199 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1200 call->del_req(cnt);
1201 }
1202 }
1203 return false;
1204 }
1205 case Op_ShenandoahCompareAndSwapP:
1206 case Op_ShenandoahCompareAndSwapN:
1207 case Op_ShenandoahWeakCompareAndSwapN:
1208 case Op_ShenandoahWeakCompareAndSwapP:
1209 case Op_ShenandoahCompareAndExchangeP:
1210 case Op_ShenandoahCompareAndExchangeN:
1211 return true;
1212 case Op_ShenandoahLoadReferenceBarrier:
1213 assert(false, "should have been expanded already");
1214 return true;
1215 default:
|
872 Node* ctrl = ac->in(TypeFunc::Control);
873 Node* mem = ac->in(TypeFunc::Memory);
874 Node* src_base = ac->in(ArrayCopyNode::Src);
875 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
876 Node* dest_base = ac->in(ArrayCopyNode::Dest);
877 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
878 Node* length = ac->in(ArrayCopyNode::Length);
879
880 Node* src = phase->basic_plus_adr(src_base, src_offset);
881 Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
882
883 if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
884 // Check if heap is has forwarded objects. If it does, we need to call into the special
885 // routine that would fix up source references before we can continue.
886
887 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
888 Node* region = new RegionNode(PATH_LIMIT);
889 Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
890
891 Node* thread = phase->transform_later(new ThreadLocalNode());
892 Node* offset = phase->MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
893 Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
894
895 uint gc_state_idx = Compile::AliasIdxRaw;
896 const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
897 DEBUG_ONLY(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
898
899 Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
900 Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
901 Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
902 Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
903
904 IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
905 Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
906 Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
907
908 // Heap is stable, no need to do anything additional
909 region->init_req(_heap_stable, stable_ctrl);
910 mem_phi->init_req(_heap_stable, mem);
911
912 // Heap is unstable, call into clone barrier stub
920
921 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
922 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
923 region->init_req(_heap_unstable, ctrl);
924 mem_phi->init_req(_heap_unstable, mem);
925
926 // Wire up the actual arraycopy stub now
927 ctrl = phase->transform_later(region);
928 mem = phase->transform_later(mem_phi);
929
930 const char* name = "arraycopy";
931 call = phase->make_leaf_call(ctrl, mem,
932 OptoRuntime::fast_arraycopy_Type(),
933 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
934 name, TypeRawPtr::BOTTOM,
935 src, dest, length
936 LP64_ONLY(COMMA phase->top()));
937 call = phase->transform_later(call);
938
939 // Hook up the whole thing into the graph
940 phase->replace_node(ac, call);
941 } else {
942 BarrierSetC2::clone_at_expansion(phase, ac);
943 }
944 }
945
946
947 // Support for macro expanded GC barriers
948 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
949 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
950 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
951 }
952 }
953
954 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
955 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
956 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
957 }
958 }
959
960 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
961 if (is_shenandoah_wb_pre_call(node)) {
962 shenandoah_eliminate_wb_pre(node, igvn);
963 }
964 if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
965 Node* shift = node->unique_out();
966 Node* addp = shift->unique_out();
967 for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
968 Node* mem = addp->last_out(j);
969 if (UseCondCardMark && mem->is_Load()) {
970 assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
971 // The load is checking if the card has been written so
972 // replace it with zero to fold the test.
973 igvn->replace_node(mem, igvn->intcon(0));
974 continue;
975 }
976 assert(mem->is_Store(), "store required");
977 igvn->replace_node(mem, mem->in(MemNode::Memory));
978 }
979 }
980 }
981
982 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
983 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
984 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
985 c = c->unique_ctrl_out();
986 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
987 c = c->unique_ctrl_out();
988 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
989 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
990 assert(iff->is_If(), "expect test");
991 if (!is_shenandoah_marking_if(igvn, iff)) {
992 c = c->unique_ctrl_out();
993 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
994 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
995 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
996 }
997 Node* cmpx = iff->in(1)->in(1);
1089 if (if_ctrl != load_ctrl) {
1090 // Skip possible CProj->NeverBranch in infinite loops
1091 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1092 && if_ctrl->in(0)->is_NeverBranch()) {
1093 if_ctrl = if_ctrl->in(0)->in(0);
1094 }
1095 }
1096 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
1097 }
1098 }
1099 }
1100 }
1101 }
1102 }
1103 }
1104 }
1105 #endif
1106
1107 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1108 if (is_shenandoah_wb_pre_call(n)) {
1109 uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain_sig()->cnt();
1110 if (n->req() > cnt) {
1111 Node* addp = n->in(cnt);
1112 if (has_only_shenandoah_wb_pre_uses(addp)) {
1113 n->del_req(cnt);
1114 if (can_reshape) {
1115 phase->is_IterGVN()->_worklist.push(addp);
1116 }
1117 return n;
1118 }
1119 }
1120 }
1121 if (n->Opcode() == Op_CmpP) {
1122 Node* in1 = n->in(1);
1123 Node* in2 = n->in(2);
1124
1125 // If one input is null, then step over the strong LRB barriers on the other input
1126 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1127 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1128 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1129 in2 = step_over_gc_barrier(in2);
1175 return nullptr;
1176 }
1177
1178 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1179 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1180 Node* u = n->fast_out(i);
1181 if (!is_shenandoah_wb_pre_call(u)) {
1182 return false;
1183 }
1184 }
1185 return n->outcnt() > 0;
1186 }
1187
1188 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1189 switch (opcode) {
1190 case Op_CallLeaf:
1191 case Op_CallLeafNoFP: {
1192 assert (n->is_Call(), "");
1193 CallNode *call = n->as_Call();
1194 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1195 uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain_sig()->cnt();
1196 if (call->req() > cnt) {
1197 assert(call->req() == cnt + 1, "only one extra input");
1198 Node *addp = call->in(cnt);
1199 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1200 call->del_req(cnt);
1201 }
1202 }
1203 return false;
1204 }
1205 case Op_ShenandoahCompareAndSwapP:
1206 case Op_ShenandoahCompareAndSwapN:
1207 case Op_ShenandoahWeakCompareAndSwapN:
1208 case Op_ShenandoahWeakCompareAndSwapP:
1209 case Op_ShenandoahCompareAndExchangeP:
1210 case Op_ShenandoahCompareAndExchangeN:
1211 return true;
1212 case Op_ShenandoahLoadReferenceBarrier:
1213 assert(false, "should have been expanded already");
1214 return true;
1215 default:
|