874 Node* ctrl = ac->in(TypeFunc::Control);
875 Node* mem = ac->in(TypeFunc::Memory);
876 Node* src_base = ac->in(ArrayCopyNode::Src);
877 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
878 Node* dest_base = ac->in(ArrayCopyNode::Dest);
879 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
880 Node* length = ac->in(ArrayCopyNode::Length);
881
882 Node* src = phase->basic_plus_adr(src_base, src_offset);
883 Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
884
885 if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
886 // Check if heap is has forwarded objects. If it does, we need to call into the special
887 // routine that would fix up source references before we can continue.
888
889 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
890 Node* region = new RegionNode(PATH_LIMIT);
891 Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
892
893 Node* thread = phase->transform_later(new ThreadLocalNode());
894 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
895 Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
896
897 uint gc_state_idx = Compile::AliasIdxRaw;
898 const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
899 DEBUG_ONLY(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
900
901 Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
902 Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
903 Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
904 Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
905
906 IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
907 Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
908 Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
909
910 // Heap is stable, no need to do anything additional
911 region->init_req(_heap_stable, stable_ctrl);
912 mem_phi->init_req(_heap_stable, mem);
913
914 // Heap is unstable, call into clone barrier stub
922
923 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
924 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
925 region->init_req(_heap_unstable, ctrl);
926 mem_phi->init_req(_heap_unstable, mem);
927
928 // Wire up the actual arraycopy stub now
929 ctrl = phase->transform_later(region);
930 mem = phase->transform_later(mem_phi);
931
932 const char* name = "arraycopy";
933 call = phase->make_leaf_call(ctrl, mem,
934 OptoRuntime::fast_arraycopy_Type(),
935 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
936 name, TypeRawPtr::BOTTOM,
937 src, dest, length
938 LP64_ONLY(COMMA phase->top()));
939 call = phase->transform_later(call);
940
941 // Hook up the whole thing into the graph
942 phase->igvn().replace_node(ac, call);
943 } else {
944 BarrierSetC2::clone_at_expansion(phase, ac);
945 }
946 }
947
948
949 // Support for macro expanded GC barriers
950 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
951 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
952 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
953 }
954 }
955
956 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
957 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
958 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
959 }
960 }
961
962 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
963 if (is_shenandoah_wb_pre_call(node)) {
964 shenandoah_eliminate_wb_pre(node, ¯o->igvn());
965 }
966 if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
967 Node* shift = node->unique_out();
968 Node* addp = shift->unique_out();
969 for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
970 Node* mem = addp->last_out(j);
971 if (UseCondCardMark && mem->is_Load()) {
972 assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
973 // The load is checking if the card has been written so
974 // replace it with zero to fold the test.
975 macro->replace_node(mem, macro->intcon(0));
976 continue;
977 }
978 assert(mem->is_Store(), "store required");
979 macro->replace_node(mem, mem->in(MemNode::Memory));
980 }
981 }
982 }
983
984 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
985 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
986 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
987 c = c->unique_ctrl_out();
988 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
989 c = c->unique_ctrl_out();
990 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
991 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
992 assert(iff->is_If(), "expect test");
993 if (!is_shenandoah_marking_if(igvn, iff)) {
994 c = c->unique_ctrl_out();
995 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
996 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
997 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
998 }
999 Node* cmpx = iff->in(1)->in(1);
1091 if (if_ctrl != load_ctrl) {
1092 // Skip possible CProj->NeverBranch in infinite loops
1093 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1094 && if_ctrl->in(0)->is_NeverBranch()) {
1095 if_ctrl = if_ctrl->in(0)->in(0);
1096 }
1097 }
1098 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
1099 }
1100 }
1101 }
1102 }
1103 }
1104 }
1105 }
1106 }
1107 #endif
1108
1109 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1110 if (is_shenandoah_wb_pre_call(n)) {
1111 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_Type()->domain()->cnt();
1112 if (n->req() > cnt) {
1113 Node* addp = n->in(cnt);
1114 if (has_only_shenandoah_wb_pre_uses(addp)) {
1115 n->del_req(cnt);
1116 if (can_reshape) {
1117 phase->is_IterGVN()->_worklist.push(addp);
1118 }
1119 return n;
1120 }
1121 }
1122 }
1123 if (n->Opcode() == Op_CmpP) {
1124 Node* in1 = n->in(1);
1125 Node* in2 = n->in(2);
1126
1127 // If one input is null, then step over the strong LRB barriers on the other input
1128 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1129 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1130 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1131 in2 = step_over_gc_barrier(in2);
1177 return nullptr;
1178 }
1179
1180 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1181 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1182 Node* u = n->fast_out(i);
1183 if (!is_shenandoah_wb_pre_call(u)) {
1184 return false;
1185 }
1186 }
1187 return n->outcnt() > 0;
1188 }
1189
1190 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1191 switch (opcode) {
1192 case Op_CallLeaf:
1193 case Op_CallLeafNoFP: {
1194 assert (n->is_Call(), "");
1195 CallNode *call = n->as_Call();
1196 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1197 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_Type()->domain()->cnt();
1198 if (call->req() > cnt) {
1199 assert(call->req() == cnt + 1, "only one extra input");
1200 Node *addp = call->in(cnt);
1201 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1202 call->del_req(cnt);
1203 }
1204 }
1205 return false;
1206 }
1207 case Op_ShenandoahCompareAndSwapP:
1208 case Op_ShenandoahCompareAndSwapN:
1209 case Op_ShenandoahWeakCompareAndSwapN:
1210 case Op_ShenandoahWeakCompareAndSwapP:
1211 case Op_ShenandoahCompareAndExchangeP:
1212 case Op_ShenandoahCompareAndExchangeN:
1213 return true;
1214 case Op_ShenandoahLoadReferenceBarrier:
1215 assert(false, "should have been expanded already");
1216 return true;
1217 default:
|
874 Node* ctrl = ac->in(TypeFunc::Control);
875 Node* mem = ac->in(TypeFunc::Memory);
876 Node* src_base = ac->in(ArrayCopyNode::Src);
877 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
878 Node* dest_base = ac->in(ArrayCopyNode::Dest);
879 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
880 Node* length = ac->in(ArrayCopyNode::Length);
881
882 Node* src = phase->basic_plus_adr(src_base, src_offset);
883 Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
884
885 if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
886 // Check if heap is has forwarded objects. If it does, we need to call into the special
887 // routine that would fix up source references before we can continue.
888
889 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
890 Node* region = new RegionNode(PATH_LIMIT);
891 Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
892
893 Node* thread = phase->transform_later(new ThreadLocalNode());
894 Node* offset = phase->MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
895 Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
896
897 uint gc_state_idx = Compile::AliasIdxRaw;
898 const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
899 DEBUG_ONLY(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
900
901 Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
902 Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
903 Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
904 Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
905
906 IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
907 Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
908 Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
909
910 // Heap is stable, no need to do anything additional
911 region->init_req(_heap_stable, stable_ctrl);
912 mem_phi->init_req(_heap_stable, mem);
913
914 // Heap is unstable, call into clone barrier stub
922
923 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
924 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
925 region->init_req(_heap_unstable, ctrl);
926 mem_phi->init_req(_heap_unstable, mem);
927
928 // Wire up the actual arraycopy stub now
929 ctrl = phase->transform_later(region);
930 mem = phase->transform_later(mem_phi);
931
932 const char* name = "arraycopy";
933 call = phase->make_leaf_call(ctrl, mem,
934 OptoRuntime::fast_arraycopy_Type(),
935 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
936 name, TypeRawPtr::BOTTOM,
937 src, dest, length
938 LP64_ONLY(COMMA phase->top()));
939 call = phase->transform_later(call);
940
941 // Hook up the whole thing into the graph
942 phase->replace_node(ac, call);
943 } else {
944 BarrierSetC2::clone_at_expansion(phase, ac);
945 }
946 }
947
948
949 // Support for macro expanded GC barriers
950 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
951 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
952 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
953 }
954 }
955
956 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
957 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
958 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
959 }
960 }
961
962 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
963 if (is_shenandoah_wb_pre_call(node)) {
964 shenandoah_eliminate_wb_pre(node, igvn);
965 }
966 if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
967 Node* shift = node->unique_out();
968 Node* addp = shift->unique_out();
969 for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
970 Node* mem = addp->last_out(j);
971 if (UseCondCardMark && mem->is_Load()) {
972 assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
973 // The load is checking if the card has been written so
974 // replace it with zero to fold the test.
975 igvn->replace_node(mem, igvn->intcon(0));
976 continue;
977 }
978 assert(mem->is_Store(), "store required");
979 igvn->replace_node(mem, mem->in(MemNode::Memory));
980 }
981 }
982 }
983
984 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
985 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
986 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
987 c = c->unique_ctrl_out();
988 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
989 c = c->unique_ctrl_out();
990 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
991 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
992 assert(iff->is_If(), "expect test");
993 if (!is_shenandoah_marking_if(igvn, iff)) {
994 c = c->unique_ctrl_out();
995 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
996 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
997 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
998 }
999 Node* cmpx = iff->in(1)->in(1);
1091 if (if_ctrl != load_ctrl) {
1092 // Skip possible CProj->NeverBranch in infinite loops
1093 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1094 && if_ctrl->in(0)->is_NeverBranch()) {
1095 if_ctrl = if_ctrl->in(0)->in(0);
1096 }
1097 }
1098 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
1099 }
1100 }
1101 }
1102 }
1103 }
1104 }
1105 }
1106 }
1107 #endif
1108
1109 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1110 if (is_shenandoah_wb_pre_call(n)) {
1111 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_Type()->domain_sig()->cnt();
1112 if (n->req() > cnt) {
1113 Node* addp = n->in(cnt);
1114 if (has_only_shenandoah_wb_pre_uses(addp)) {
1115 n->del_req(cnt);
1116 if (can_reshape) {
1117 phase->is_IterGVN()->_worklist.push(addp);
1118 }
1119 return n;
1120 }
1121 }
1122 }
1123 if (n->Opcode() == Op_CmpP) {
1124 Node* in1 = n->in(1);
1125 Node* in2 = n->in(2);
1126
1127 // If one input is null, then step over the strong LRB barriers on the other input
1128 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1129 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1130 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1131 in2 = step_over_gc_barrier(in2);
1177 return nullptr;
1178 }
1179
1180 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1181 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1182 Node* u = n->fast_out(i);
1183 if (!is_shenandoah_wb_pre_call(u)) {
1184 return false;
1185 }
1186 }
1187 return n->outcnt() > 0;
1188 }
1189
1190 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1191 switch (opcode) {
1192 case Op_CallLeaf:
1193 case Op_CallLeafNoFP: {
1194 assert (n->is_Call(), "");
1195 CallNode *call = n->as_Call();
1196 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1197 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_Type()->domain_sig()->cnt();
1198 if (call->req() > cnt) {
1199 assert(call->req() == cnt + 1, "only one extra input");
1200 Node *addp = call->in(cnt);
1201 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1202 call->del_req(cnt);
1203 }
1204 }
1205 return false;
1206 }
1207 case Op_ShenandoahCompareAndSwapP:
1208 case Op_ShenandoahCompareAndSwapN:
1209 case Op_ShenandoahWeakCompareAndSwapN:
1210 case Op_ShenandoahWeakCompareAndSwapP:
1211 case Op_ShenandoahCompareAndExchangeP:
1212 case Op_ShenandoahCompareAndExchangeN:
1213 return true;
1214 case Op_ShenandoahLoadReferenceBarrier:
1215 assert(false, "should have been expanded already");
1216 return true;
1217 default:
|