811 Node* ctrl = ac->in(TypeFunc::Control);
812 Node* mem = ac->in(TypeFunc::Memory);
813 Node* src_base = ac->in(ArrayCopyNode::Src);
814 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
815 Node* dest_base = ac->in(ArrayCopyNode::Dest);
816 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
817 Node* length = ac->in(ArrayCopyNode::Length);
818
819 Node* src = phase->basic_plus_adr(src_base, src_offset);
820 Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
821
822 if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
823 // Check if heap is has forwarded objects. If it does, we need to call into the special
824 // routine that would fix up source references before we can continue.
825
826 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
827 Node* region = new RegionNode(PATH_LIMIT);
828 Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
829
830 Node* thread = phase->transform_later(new ThreadLocalNode());
831 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
832 Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
833
834 uint gc_state_idx = Compile::AliasIdxRaw;
835 const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
836 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
837
838 Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
839 int flags = ShenandoahHeap::HAS_FORWARDED;
840 if (ShenandoahIUBarrier) {
841 flags |= ShenandoahHeap::MARKING;
842 }
843 Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags)));
844 Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
845 Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
846
847 IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
848 Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
849 Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
850
851 // Heap is stable, no need to do anything additional
863
864 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
865 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
866 region->init_req(_heap_unstable, ctrl);
867 mem_phi->init_req(_heap_unstable, mem);
868
869 // Wire up the actual arraycopy stub now
870 ctrl = phase->transform_later(region);
871 mem = phase->transform_later(mem_phi);
872
873 const char* name = "arraycopy";
874 call = phase->make_leaf_call(ctrl, mem,
875 OptoRuntime::fast_arraycopy_Type(),
876 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
877 name, TypeRawPtr::BOTTOM,
878 src, dest, length
879 LP64_ONLY(COMMA phase->top()));
880 call = phase->transform_later(call);
881
882 // Hook up the whole thing into the graph
883 phase->igvn().replace_node(ac, call);
884 } else {
885 BarrierSetC2::clone_at_expansion(phase, ac);
886 }
887 }
888
889
890 // Support for macro expanded GC barriers
891 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
892 if (node->Opcode() == Op_ShenandoahIUBarrier) {
893 state()->add_iu_barrier((ShenandoahIUBarrierNode*) node);
894 }
895 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
896 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
897 }
898 }
899
900 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
901 if (node->Opcode() == Op_ShenandoahIUBarrier) {
902 state()->remove_iu_barrier((ShenandoahIUBarrierNode*) node);
903 }
904 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
905 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
906 }
907 }
908
909 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
910 if (is_shenandoah_wb_pre_call(n)) {
911 shenandoah_eliminate_wb_pre(n, ¯o->igvn());
912 }
913 }
914
915 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
916 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
917 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
918 c = c->unique_ctrl_out();
919 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
920 c = c->unique_ctrl_out();
921 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
922 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
923 assert(iff->is_If(), "expect test");
924 if (!is_shenandoah_marking_if(igvn, iff)) {
925 c = c->unique_ctrl_out();
926 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
927 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
928 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
929 }
930 Node* cmpx = iff->in(1)->in(1);
931 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
1024 if (if_ctrl != load_ctrl) {
1025 // Skip possible CProj->NeverBranch in infinite loops
1026 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1027 && if_ctrl->in(0)->is_NeverBranch()) {
1028 if_ctrl = if_ctrl->in(0)->in(0);
1029 }
1030 }
1031 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
1032 }
1033 }
1034 }
1035 }
1036 }
1037 }
1038 }
1039 }
1040 #endif
1041
1042 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1043 if (is_shenandoah_wb_pre_call(n)) {
1044 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
1045 if (n->req() > cnt) {
1046 Node* addp = n->in(cnt);
1047 if (has_only_shenandoah_wb_pre_uses(addp)) {
1048 n->del_req(cnt);
1049 if (can_reshape) {
1050 phase->is_IterGVN()->_worklist.push(addp);
1051 }
1052 return n;
1053 }
1054 }
1055 }
1056 if (n->Opcode() == Op_CmpP) {
1057 Node* in1 = n->in(1);
1058 Node* in2 = n->in(2);
1059
1060 // If one input is null, then step over the strong LRB barriers on the other input
1061 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1062 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1063 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1064 in2 = step_over_gc_barrier(in2);
1110 return nullptr;
1111 }
1112
1113 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1114 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1115 Node* u = n->fast_out(i);
1116 if (!is_shenandoah_wb_pre_call(u)) {
1117 return false;
1118 }
1119 }
1120 return n->outcnt() > 0;
1121 }
1122
1123 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1124 switch (opcode) {
1125 case Op_CallLeaf:
1126 case Op_CallLeafNoFP: {
1127 assert (n->is_Call(), "");
1128 CallNode *call = n->as_Call();
1129 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1130 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
1131 if (call->req() > cnt) {
1132 assert(call->req() == cnt + 1, "only one extra input");
1133 Node *addp = call->in(cnt);
1134 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1135 call->del_req(cnt);
1136 }
1137 }
1138 return false;
1139 }
1140 case Op_ShenandoahCompareAndSwapP:
1141 case Op_ShenandoahCompareAndSwapN:
1142 case Op_ShenandoahWeakCompareAndSwapN:
1143 case Op_ShenandoahWeakCompareAndSwapP:
1144 case Op_ShenandoahCompareAndExchangeP:
1145 case Op_ShenandoahCompareAndExchangeN:
1146 return true;
1147 case Op_ShenandoahLoadReferenceBarrier:
1148 assert(false, "should have been expanded already");
1149 return true;
1150 default:
|
811 Node* ctrl = ac->in(TypeFunc::Control);
812 Node* mem = ac->in(TypeFunc::Memory);
813 Node* src_base = ac->in(ArrayCopyNode::Src);
814 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
815 Node* dest_base = ac->in(ArrayCopyNode::Dest);
816 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
817 Node* length = ac->in(ArrayCopyNode::Length);
818
819 Node* src = phase->basic_plus_adr(src_base, src_offset);
820 Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
821
822 if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
823 // Check if heap is has forwarded objects. If it does, we need to call into the special
824 // routine that would fix up source references before we can continue.
825
826 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
827 Node* region = new RegionNode(PATH_LIMIT);
828 Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
829
830 Node* thread = phase->transform_later(new ThreadLocalNode());
831 Node* offset = phase->MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
832 Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
833
834 uint gc_state_idx = Compile::AliasIdxRaw;
835 const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
836 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
837
838 Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
839 int flags = ShenandoahHeap::HAS_FORWARDED;
840 if (ShenandoahIUBarrier) {
841 flags |= ShenandoahHeap::MARKING;
842 }
843 Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags)));
844 Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
845 Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
846
847 IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
848 Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
849 Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
850
851 // Heap is stable, no need to do anything additional
863
864 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
865 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
866 region->init_req(_heap_unstable, ctrl);
867 mem_phi->init_req(_heap_unstable, mem);
868
869 // Wire up the actual arraycopy stub now
870 ctrl = phase->transform_later(region);
871 mem = phase->transform_later(mem_phi);
872
873 const char* name = "arraycopy";
874 call = phase->make_leaf_call(ctrl, mem,
875 OptoRuntime::fast_arraycopy_Type(),
876 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
877 name, TypeRawPtr::BOTTOM,
878 src, dest, length
879 LP64_ONLY(COMMA phase->top()));
880 call = phase->transform_later(call);
881
882 // Hook up the whole thing into the graph
883 phase->replace_node(ac, call);
884 } else {
885 BarrierSetC2::clone_at_expansion(phase, ac);
886 }
887 }
888
889
890 // Support for macro expanded GC barriers
891 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
892 if (node->Opcode() == Op_ShenandoahIUBarrier) {
893 state()->add_iu_barrier((ShenandoahIUBarrierNode*) node);
894 }
895 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
896 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
897 }
898 }
899
900 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
901 if (node->Opcode() == Op_ShenandoahIUBarrier) {
902 state()->remove_iu_barrier((ShenandoahIUBarrierNode*) node);
903 }
904 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
905 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
906 }
907 }
908
909 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* n) const {
910 if (is_shenandoah_wb_pre_call(n)) {
911 shenandoah_eliminate_wb_pre(n, igvn);
912 }
913 }
914
915 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
916 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
917 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
918 c = c->unique_ctrl_out();
919 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
920 c = c->unique_ctrl_out();
921 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
922 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
923 assert(iff->is_If(), "expect test");
924 if (!is_shenandoah_marking_if(igvn, iff)) {
925 c = c->unique_ctrl_out();
926 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
927 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
928 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
929 }
930 Node* cmpx = iff->in(1)->in(1);
931 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
1024 if (if_ctrl != load_ctrl) {
1025 // Skip possible CProj->NeverBranch in infinite loops
1026 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1027 && if_ctrl->in(0)->is_NeverBranch()) {
1028 if_ctrl = if_ctrl->in(0)->in(0);
1029 }
1030 }
1031 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
1032 }
1033 }
1034 }
1035 }
1036 }
1037 }
1038 }
1039 }
1040 #endif
1041
1042 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1043 if (is_shenandoah_wb_pre_call(n)) {
1044 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain_sig()->cnt();
1045 if (n->req() > cnt) {
1046 Node* addp = n->in(cnt);
1047 if (has_only_shenandoah_wb_pre_uses(addp)) {
1048 n->del_req(cnt);
1049 if (can_reshape) {
1050 phase->is_IterGVN()->_worklist.push(addp);
1051 }
1052 return n;
1053 }
1054 }
1055 }
1056 if (n->Opcode() == Op_CmpP) {
1057 Node* in1 = n->in(1);
1058 Node* in2 = n->in(2);
1059
1060 // If one input is null, then step over the strong LRB barriers on the other input
1061 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1062 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1063 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1064 in2 = step_over_gc_barrier(in2);
1110 return nullptr;
1111 }
1112
1113 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1114 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1115 Node* u = n->fast_out(i);
1116 if (!is_shenandoah_wb_pre_call(u)) {
1117 return false;
1118 }
1119 }
1120 return n->outcnt() > 0;
1121 }
1122
1123 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1124 switch (opcode) {
1125 case Op_CallLeaf:
1126 case Op_CallLeafNoFP: {
1127 assert (n->is_Call(), "");
1128 CallNode *call = n->as_Call();
1129 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1130 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain_sig()->cnt();
1131 if (call->req() > cnt) {
1132 assert(call->req() == cnt + 1, "only one extra input");
1133 Node *addp = call->in(cnt);
1134 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1135 call->del_req(cnt);
1136 }
1137 }
1138 return false;
1139 }
1140 case Op_ShenandoahCompareAndSwapP:
1141 case Op_ShenandoahCompareAndSwapN:
1142 case Op_ShenandoahWeakCompareAndSwapN:
1143 case Op_ShenandoahWeakCompareAndSwapP:
1144 case Op_ShenandoahCompareAndExchangeP:
1145 case Op_ShenandoahCompareAndExchangeN:
1146 return true;
1147 case Op_ShenandoahLoadReferenceBarrier:
1148 assert(false, "should have been expanded already");
1149 return true;
1150 default:
|