767 Node* ctrl = ac->in(TypeFunc::Control);
768 Node* mem = ac->in(TypeFunc::Memory);
769 Node* src_base = ac->in(ArrayCopyNode::Src);
770 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
771 Node* dest_base = ac->in(ArrayCopyNode::Dest);
772 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
773 Node* length = ac->in(ArrayCopyNode::Length);
774
775 Node* src = phase->basic_plus_adr(src_base, src_offset);
776 Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
777
778 if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
779 // Check if heap is has forwarded objects. If it does, we need to call into the special
780 // routine that would fix up source references before we can continue.
781
782 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
783 Node* region = new RegionNode(PATH_LIMIT);
784 Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
785
786 Node* thread = phase->transform_later(new ThreadLocalNode());
787 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
788 Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
789
790 uint gc_state_idx = Compile::AliasIdxRaw;
791 const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
792 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
793
794 Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
795 Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
796 Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
797 Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
798
799 IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
800 Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
801 Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
802
803 // Heap is stable, no need to do anything additional
804 region->init_req(_heap_stable, stable_ctrl);
805 mem_phi->init_req(_heap_stable, mem);
806
807 // Heap is unstable, call into clone barrier stub
815
816 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
817 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
818 region->init_req(_heap_unstable, ctrl);
819 mem_phi->init_req(_heap_unstable, mem);
820
821 // Wire up the actual arraycopy stub now
822 ctrl = phase->transform_later(region);
823 mem = phase->transform_later(mem_phi);
824
825 const char* name = "arraycopy";
826 call = phase->make_leaf_call(ctrl, mem,
827 OptoRuntime::fast_arraycopy_Type(),
828 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
829 name, TypeRawPtr::BOTTOM,
830 src, dest, length
831 LP64_ONLY(COMMA phase->top()));
832 call = phase->transform_later(call);
833
834 // Hook up the whole thing into the graph
835 phase->igvn().replace_node(ac, call);
836 } else {
837 BarrierSetC2::clone_at_expansion(phase, ac);
838 }
839 }
840
841
842 // Support for macro expanded GC barriers
843 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
844 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
845 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
846 }
847 }
848
849 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
850 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
851 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
852 }
853 }
854
855 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
856 if (is_shenandoah_wb_pre_call(n)) {
857 shenandoah_eliminate_wb_pre(n, ¯o->igvn());
858 }
859 }
860
861 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
862 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
863 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
864 c = c->unique_ctrl_out();
865 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
866 c = c->unique_ctrl_out();
867 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
868 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
869 assert(iff->is_If(), "expect test");
870 if (!is_shenandoah_marking_if(igvn, iff)) {
871 c = c->unique_ctrl_out();
872 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
873 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
874 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
875 }
876 Node* cmpx = iff->in(1)->in(1);
877 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
968 if (if_ctrl != load_ctrl) {
969 // Skip possible CProj->NeverBranch in infinite loops
970 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
971 && if_ctrl->in(0)->is_NeverBranch()) {
972 if_ctrl = if_ctrl->in(0)->in(0);
973 }
974 }
975 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
976 }
977 }
978 }
979 }
980 }
981 }
982 }
983 }
984 #endif
985
986 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
987 if (is_shenandoah_wb_pre_call(n)) {
988 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_Type()->domain()->cnt();
989 if (n->req() > cnt) {
990 Node* addp = n->in(cnt);
991 if (has_only_shenandoah_wb_pre_uses(addp)) {
992 n->del_req(cnt);
993 if (can_reshape) {
994 phase->is_IterGVN()->_worklist.push(addp);
995 }
996 return n;
997 }
998 }
999 }
1000 if (n->Opcode() == Op_CmpP) {
1001 Node* in1 = n->in(1);
1002 Node* in2 = n->in(2);
1003
1004 // If one input is null, then step over the strong LRB barriers on the other input
1005 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1006 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1007 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1008 in2 = step_over_gc_barrier(in2);
1054 return nullptr;
1055 }
1056
1057 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1058 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1059 Node* u = n->fast_out(i);
1060 if (!is_shenandoah_wb_pre_call(u)) {
1061 return false;
1062 }
1063 }
1064 return n->outcnt() > 0;
1065 }
1066
1067 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1068 switch (opcode) {
1069 case Op_CallLeaf:
1070 case Op_CallLeafNoFP: {
1071 assert (n->is_Call(), "");
1072 CallNode *call = n->as_Call();
1073 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1074 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_Type()->domain()->cnt();
1075 if (call->req() > cnt) {
1076 assert(call->req() == cnt + 1, "only one extra input");
1077 Node *addp = call->in(cnt);
1078 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1079 call->del_req(cnt);
1080 }
1081 }
1082 return false;
1083 }
1084 case Op_ShenandoahCompareAndSwapP:
1085 case Op_ShenandoahCompareAndSwapN:
1086 case Op_ShenandoahWeakCompareAndSwapN:
1087 case Op_ShenandoahWeakCompareAndSwapP:
1088 case Op_ShenandoahCompareAndExchangeP:
1089 case Op_ShenandoahCompareAndExchangeN:
1090 return true;
1091 case Op_ShenandoahLoadReferenceBarrier:
1092 assert(false, "should have been expanded already");
1093 return true;
1094 default:
|
767 Node* ctrl = ac->in(TypeFunc::Control);
768 Node* mem = ac->in(TypeFunc::Memory);
769 Node* src_base = ac->in(ArrayCopyNode::Src);
770 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
771 Node* dest_base = ac->in(ArrayCopyNode::Dest);
772 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
773 Node* length = ac->in(ArrayCopyNode::Length);
774
775 Node* src = phase->basic_plus_adr(src_base, src_offset);
776 Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
777
778 if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
779 // Check if heap is has forwarded objects. If it does, we need to call into the special
780 // routine that would fix up source references before we can continue.
781
782 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
783 Node* region = new RegionNode(PATH_LIMIT);
784 Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
785
786 Node* thread = phase->transform_later(new ThreadLocalNode());
787 Node* offset = phase->MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
788 Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
789
790 uint gc_state_idx = Compile::AliasIdxRaw;
791 const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
792 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
793
794 Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
795 Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
796 Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
797 Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
798
799 IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
800 Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
801 Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
802
803 // Heap is stable, no need to do anything additional
804 region->init_req(_heap_stable, stable_ctrl);
805 mem_phi->init_req(_heap_stable, mem);
806
807 // Heap is unstable, call into clone barrier stub
815
816 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
817 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
818 region->init_req(_heap_unstable, ctrl);
819 mem_phi->init_req(_heap_unstable, mem);
820
821 // Wire up the actual arraycopy stub now
822 ctrl = phase->transform_later(region);
823 mem = phase->transform_later(mem_phi);
824
825 const char* name = "arraycopy";
826 call = phase->make_leaf_call(ctrl, mem,
827 OptoRuntime::fast_arraycopy_Type(),
828 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
829 name, TypeRawPtr::BOTTOM,
830 src, dest, length
831 LP64_ONLY(COMMA phase->top()));
832 call = phase->transform_later(call);
833
834 // Hook up the whole thing into the graph
835 phase->replace_node(ac, call);
836 } else {
837 BarrierSetC2::clone_at_expansion(phase, ac);
838 }
839 }
840
841
842 // Support for macro expanded GC barriers
843 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
844 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
845 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
846 }
847 }
848
849 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
850 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
851 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
852 }
853 }
854
855 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* n) const {
856 if (is_shenandoah_wb_pre_call(n)) {
857 shenandoah_eliminate_wb_pre(n, igvn);
858 }
859 }
860
861 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
862 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
863 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
864 c = c->unique_ctrl_out();
865 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
866 c = c->unique_ctrl_out();
867 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
868 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
869 assert(iff->is_If(), "expect test");
870 if (!is_shenandoah_marking_if(igvn, iff)) {
871 c = c->unique_ctrl_out();
872 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
873 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
874 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
875 }
876 Node* cmpx = iff->in(1)->in(1);
877 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
968 if (if_ctrl != load_ctrl) {
969 // Skip possible CProj->NeverBranch in infinite loops
970 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
971 && if_ctrl->in(0)->is_NeverBranch()) {
972 if_ctrl = if_ctrl->in(0)->in(0);
973 }
974 }
975 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
976 }
977 }
978 }
979 }
980 }
981 }
982 }
983 }
984 #endif
985
986 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
987 if (is_shenandoah_wb_pre_call(n)) {
988 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_Type()->domain_sig()->cnt();
989 if (n->req() > cnt) {
990 Node* addp = n->in(cnt);
991 if (has_only_shenandoah_wb_pre_uses(addp)) {
992 n->del_req(cnt);
993 if (can_reshape) {
994 phase->is_IterGVN()->_worklist.push(addp);
995 }
996 return n;
997 }
998 }
999 }
1000 if (n->Opcode() == Op_CmpP) {
1001 Node* in1 = n->in(1);
1002 Node* in2 = n->in(2);
1003
1004 // If one input is null, then step over the strong LRB barriers on the other input
1005 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1006 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1007 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1008 in2 = step_over_gc_barrier(in2);
1054 return nullptr;
1055 }
1056
1057 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1058 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1059 Node* u = n->fast_out(i);
1060 if (!is_shenandoah_wb_pre_call(u)) {
1061 return false;
1062 }
1063 }
1064 return n->outcnt() > 0;
1065 }
1066
1067 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1068 switch (opcode) {
1069 case Op_CallLeaf:
1070 case Op_CallLeafNoFP: {
1071 assert (n->is_Call(), "");
1072 CallNode *call = n->as_Call();
1073 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1074 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_Type()->domain_sig()->cnt();
1075 if (call->req() > cnt) {
1076 assert(call->req() == cnt + 1, "only one extra input");
1077 Node *addp = call->in(cnt);
1078 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1079 call->del_req(cnt);
1080 }
1081 }
1082 return false;
1083 }
1084 case Op_ShenandoahCompareAndSwapP:
1085 case Op_ShenandoahCompareAndSwapN:
1086 case Op_ShenandoahWeakCompareAndSwapN:
1087 case Op_ShenandoahWeakCompareAndSwapP:
1088 case Op_ShenandoahCompareAndExchangeP:
1089 case Op_ShenandoahCompareAndExchangeN:
1090 return true;
1091 case Op_ShenandoahLoadReferenceBarrier:
1092 assert(false, "should have been expanded already");
1093 return true;
1094 default:
|