762 Node* ctrl = ac->in(TypeFunc::Control);
763 Node* mem = ac->in(TypeFunc::Memory);
764 Node* src_base = ac->in(ArrayCopyNode::Src);
765 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
766 Node* dest_base = ac->in(ArrayCopyNode::Dest);
767 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
768 Node* length = ac->in(ArrayCopyNode::Length);
769
770 Node* src = phase->basic_plus_adr(src_base, src_offset);
771 Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
772
773 if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
774 // Check if heap is has forwarded objects. If it does, we need to call into the special
775 // routine that would fix up source references before we can continue.
776
777 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
778 Node* region = new RegionNode(PATH_LIMIT);
779 Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
780
781 Node* thread = phase->transform_later(new ThreadLocalNode());
782 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
783 Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
784
785 uint gc_state_idx = Compile::AliasIdxRaw;
786 const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
787 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
788
789 Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
790 Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
791 Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
792 Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
793
794 IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
795 Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
796 Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
797
798 // Heap is stable, no need to do anything additional
799 region->init_req(_heap_stable, stable_ctrl);
800 mem_phi->init_req(_heap_stable, mem);
801
802 // Heap is unstable, call into clone barrier stub
810
811 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
812 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
813 region->init_req(_heap_unstable, ctrl);
814 mem_phi->init_req(_heap_unstable, mem);
815
816 // Wire up the actual arraycopy stub now
817 ctrl = phase->transform_later(region);
818 mem = phase->transform_later(mem_phi);
819
820 const char* name = "arraycopy";
821 call = phase->make_leaf_call(ctrl, mem,
822 OptoRuntime::fast_arraycopy_Type(),
823 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
824 name, TypeRawPtr::BOTTOM,
825 src, dest, length
826 LP64_ONLY(COMMA phase->top()));
827 call = phase->transform_later(call);
828
829 // Hook up the whole thing into the graph
830 phase->igvn().replace_node(ac, call);
831 } else {
832 BarrierSetC2::clone_at_expansion(phase, ac);
833 }
834 }
835
836
837 // Support for macro expanded GC barriers
838 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
839 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
840 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
841 }
842 }
843
844 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
845 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
846 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
847 }
848 }
849
850 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
851 if (is_shenandoah_wb_pre_call(n)) {
852 shenandoah_eliminate_wb_pre(n, ¯o->igvn());
853 }
854 }
855
856 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
857 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
858 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
859 c = c->unique_ctrl_out();
860 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
861 c = c->unique_ctrl_out();
862 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
863 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
864 assert(iff->is_If(), "expect test");
865 if (!is_shenandoah_marking_if(igvn, iff)) {
866 c = c->unique_ctrl_out();
867 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
868 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
869 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
870 }
871 Node* cmpx = iff->in(1)->in(1);
872 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
963 if (if_ctrl != load_ctrl) {
964 // Skip possible CProj->NeverBranch in infinite loops
965 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
966 && if_ctrl->in(0)->is_NeverBranch()) {
967 if_ctrl = if_ctrl->in(0)->in(0);
968 }
969 }
970 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
971 }
972 }
973 }
974 }
975 }
976 }
977 }
978 }
979 #endif
980
981 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
982 if (is_shenandoah_wb_pre_call(n)) {
983 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
984 if (n->req() > cnt) {
985 Node* addp = n->in(cnt);
986 if (has_only_shenandoah_wb_pre_uses(addp)) {
987 n->del_req(cnt);
988 if (can_reshape) {
989 phase->is_IterGVN()->_worklist.push(addp);
990 }
991 return n;
992 }
993 }
994 }
995 if (n->Opcode() == Op_CmpP) {
996 Node* in1 = n->in(1);
997 Node* in2 = n->in(2);
998
999 // If one input is null, then step over the strong LRB barriers on the other input
1000 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1001 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1002 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1003 in2 = step_over_gc_barrier(in2);
1049 return nullptr;
1050 }
1051
1052 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1053 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1054 Node* u = n->fast_out(i);
1055 if (!is_shenandoah_wb_pre_call(u)) {
1056 return false;
1057 }
1058 }
1059 return n->outcnt() > 0;
1060 }
1061
1062 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1063 switch (opcode) {
1064 case Op_CallLeaf:
1065 case Op_CallLeafNoFP: {
1066 assert (n->is_Call(), "");
1067 CallNode *call = n->as_Call();
1068 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1069 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
1070 if (call->req() > cnt) {
1071 assert(call->req() == cnt + 1, "only one extra input");
1072 Node *addp = call->in(cnt);
1073 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1074 call->del_req(cnt);
1075 }
1076 }
1077 return false;
1078 }
1079 case Op_ShenandoahCompareAndSwapP:
1080 case Op_ShenandoahCompareAndSwapN:
1081 case Op_ShenandoahWeakCompareAndSwapN:
1082 case Op_ShenandoahWeakCompareAndSwapP:
1083 case Op_ShenandoahCompareAndExchangeP:
1084 case Op_ShenandoahCompareAndExchangeN:
1085 return true;
1086 case Op_ShenandoahLoadReferenceBarrier:
1087 assert(false, "should have been expanded already");
1088 return true;
1089 default:
|
762 Node* ctrl = ac->in(TypeFunc::Control);
763 Node* mem = ac->in(TypeFunc::Memory);
764 Node* src_base = ac->in(ArrayCopyNode::Src);
765 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
766 Node* dest_base = ac->in(ArrayCopyNode::Dest);
767 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
768 Node* length = ac->in(ArrayCopyNode::Length);
769
770 Node* src = phase->basic_plus_adr(src_base, src_offset);
771 Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
772
773 if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
774 // Check if heap is has forwarded objects. If it does, we need to call into the special
775 // routine that would fix up source references before we can continue.
776
777 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
778 Node* region = new RegionNode(PATH_LIMIT);
779 Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
780
781 Node* thread = phase->transform_later(new ThreadLocalNode());
782 Node* offset = phase->MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
783 Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
784
785 uint gc_state_idx = Compile::AliasIdxRaw;
786 const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
787 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
788
789 Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
790 Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
791 Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
792 Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
793
794 IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
795 Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
796 Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
797
798 // Heap is stable, no need to do anything additional
799 region->init_req(_heap_stable, stable_ctrl);
800 mem_phi->init_req(_heap_stable, mem);
801
802 // Heap is unstable, call into clone barrier stub
810
811 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
812 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
813 region->init_req(_heap_unstable, ctrl);
814 mem_phi->init_req(_heap_unstable, mem);
815
816 // Wire up the actual arraycopy stub now
817 ctrl = phase->transform_later(region);
818 mem = phase->transform_later(mem_phi);
819
820 const char* name = "arraycopy";
821 call = phase->make_leaf_call(ctrl, mem,
822 OptoRuntime::fast_arraycopy_Type(),
823 phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
824 name, TypeRawPtr::BOTTOM,
825 src, dest, length
826 LP64_ONLY(COMMA phase->top()));
827 call = phase->transform_later(call);
828
829 // Hook up the whole thing into the graph
830 phase->replace_node(ac, call);
831 } else {
832 BarrierSetC2::clone_at_expansion(phase, ac);
833 }
834 }
835
836
837 // Support for macro expanded GC barriers
838 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
839 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
840 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
841 }
842 }
843
844 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
845 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
846 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
847 }
848 }
849
850 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* n) const {
851 if (is_shenandoah_wb_pre_call(n)) {
852 shenandoah_eliminate_wb_pre(n, igvn);
853 }
854 }
855
856 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
857 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
858 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
859 c = c->unique_ctrl_out();
860 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
861 c = c->unique_ctrl_out();
862 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
863 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
864 assert(iff->is_If(), "expect test");
865 if (!is_shenandoah_marking_if(igvn, iff)) {
866 c = c->unique_ctrl_out();
867 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
868 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
869 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
870 }
871 Node* cmpx = iff->in(1)->in(1);
872 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
963 if (if_ctrl != load_ctrl) {
964 // Skip possible CProj->NeverBranch in infinite loops
965 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
966 && if_ctrl->in(0)->is_NeverBranch()) {
967 if_ctrl = if_ctrl->in(0)->in(0);
968 }
969 }
970 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
971 }
972 }
973 }
974 }
975 }
976 }
977 }
978 }
979 #endif
980
981 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
982 if (is_shenandoah_wb_pre_call(n)) {
983 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain_sig()->cnt();
984 if (n->req() > cnt) {
985 Node* addp = n->in(cnt);
986 if (has_only_shenandoah_wb_pre_uses(addp)) {
987 n->del_req(cnt);
988 if (can_reshape) {
989 phase->is_IterGVN()->_worklist.push(addp);
990 }
991 return n;
992 }
993 }
994 }
995 if (n->Opcode() == Op_CmpP) {
996 Node* in1 = n->in(1);
997 Node* in2 = n->in(2);
998
999 // If one input is null, then step over the strong LRB barriers on the other input
1000 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1001 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1002 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1003 in2 = step_over_gc_barrier(in2);
1049 return nullptr;
1050 }
1051
1052 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1053 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1054 Node* u = n->fast_out(i);
1055 if (!is_shenandoah_wb_pre_call(u)) {
1056 return false;
1057 }
1058 }
1059 return n->outcnt() > 0;
1060 }
1061
1062 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1063 switch (opcode) {
1064 case Op_CallLeaf:
1065 case Op_CallLeafNoFP: {
1066 assert (n->is_Call(), "");
1067 CallNode *call = n->as_Call();
1068 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1069 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain_sig()->cnt();
1070 if (call->req() > cnt) {
1071 assert(call->req() == cnt + 1, "only one extra input");
1072 Node *addp = call->in(cnt);
1073 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1074 call->del_req(cnt);
1075 }
1076 }
1077 return false;
1078 }
1079 case Op_ShenandoahCompareAndSwapP:
1080 case Op_ShenandoahCompareAndSwapN:
1081 case Op_ShenandoahWeakCompareAndSwapN:
1082 case Op_ShenandoahWeakCompareAndSwapP:
1083 case Op_ShenandoahCompareAndExchangeP:
1084 case Op_ShenandoahCompareAndExchangeN:
1085 return true;
1086 case Op_ShenandoahLoadReferenceBarrier:
1087 assert(false, "should have been expanded already");
1088 return true;
1089 default:
|