810 Node* ctrl = ac->in(TypeFunc::Control);
811 Node* mem = ac->in(TypeFunc::Memory);
812 Node* src_base = ac->in(ArrayCopyNode::Src);
813 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
814 Node* dest_base = ac->in(ArrayCopyNode::Dest);
815 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
816 Node* length = ac->in(ArrayCopyNode::Length);
817
818 Node* src = phase->basic_plus_adr(src_base, src_offset);
819 Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
820
821 if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
822 // Check if heap is has forwarded objects. If it does, we need to call into the special
823 // routine that would fix up source references before we can continue.
824
825 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
826 Node* region = new RegionNode(PATH_LIMIT);
827 Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
828
829 Node* thread = phase->transform_later(new ThreadLocalNode());
830 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
831 Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
832
833 uint gc_state_idx = Compile::AliasIdxRaw;
834 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
835 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
836
837 Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
838 int flags = ShenandoahHeap::HAS_FORWARDED;
839 if (ShenandoahIUBarrier) {
840 flags |= ShenandoahHeap::MARKING;
841 }
842 Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags)));
843 Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
844 Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
845
846 IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
847 Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
848 Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
849
850 // Heap is stable, no need to do anything additional
862
863 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
864 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
865 region->init_req(_heap_unstable, ctrl);
866 mem_phi->init_req(_heap_unstable, mem);
867
868 // Wire up the actual arraycopy stub now
869 ctrl = phase->transform_later(region);
870 mem = phase->transform_later(mem_phi);
871
872 const char* name = "arraycopy";
873 call = phase->make_leaf_call(ctrl, mem,
874 OptoRuntime::fast_arraycopy_Type(),
875 phase->basictype2arraycopy(T_LONG, NULL, NULL, true, name, true),
876 name, TypeRawPtr::BOTTOM,
877 src, dest, length
878 LP64_ONLY(COMMA phase->top()));
879 call = phase->transform_later(call);
880
881 // Hook up the whole thing into the graph
882 phase->igvn().replace_node(ac, call);
883 } else {
884 BarrierSetC2::clone_at_expansion(phase, ac);
885 }
886 }
887
888
889 // Support for macro expanded GC barriers
890 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
891 if (node->Opcode() == Op_ShenandoahIUBarrier) {
892 state()->add_iu_barrier((ShenandoahIUBarrierNode*) node);
893 }
894 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
895 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
896 }
897 }
898
899 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
900 if (node->Opcode() == Op_ShenandoahIUBarrier) {
901 state()->remove_iu_barrier((ShenandoahIUBarrierNode*) node);
902 }
903 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
904 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
905 }
906 }
907
908 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
909 if (is_shenandoah_wb_pre_call(n)) {
910 shenandoah_eliminate_wb_pre(n, ¯o->igvn());
911 }
912 }
913
914 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
915 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
916 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
917 c = c->unique_ctrl_out();
918 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
919 c = c->unique_ctrl_out();
920 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
921 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
922 assert(iff->is_If(), "expect test");
923 if (!is_shenandoah_marking_if(igvn, iff)) {
924 c = c->unique_ctrl_out();
925 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
926 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
927 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
928 }
929 Node* cmpx = iff->in(1)->in(1);
930 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
1023 if (if_ctrl != load_ctrl) {
1024 // Skip possible CProj->NeverBranch in infinite loops
1025 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1026 && (if_ctrl->in(0)->is_MultiBranch() && if_ctrl->in(0)->Opcode() == Op_NeverBranch)) {
1027 if_ctrl = if_ctrl->in(0)->in(0);
1028 }
1029 }
1030 assert(load_ctrl != NULL && if_ctrl == load_ctrl, "controls must match");
1031 }
1032 }
1033 }
1034 }
1035 }
1036 }
1037 }
1038 }
1039 #endif
1040
1041 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1042 if (is_shenandoah_wb_pre_call(n)) {
1043 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
1044 if (n->req() > cnt) {
1045 Node* addp = n->in(cnt);
1046 if (has_only_shenandoah_wb_pre_uses(addp)) {
1047 n->del_req(cnt);
1048 if (can_reshape) {
1049 phase->is_IterGVN()->_worklist.push(addp);
1050 }
1051 return n;
1052 }
1053 }
1054 }
1055 if (n->Opcode() == Op_CmpP) {
1056 Node* in1 = n->in(1);
1057 Node* in2 = n->in(2);
1058
1059 // If one input is NULL, then step over the strong LRB barriers on the other input
1060 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1061 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1062 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1063 in2 = step_over_gc_barrier(in2);
1108 return NULL;
1109 }
1110
1111 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1112 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1113 Node* u = n->fast_out(i);
1114 if (!is_shenandoah_wb_pre_call(u)) {
1115 return false;
1116 }
1117 }
1118 return n->outcnt() > 0;
1119 }
1120
1121 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
1122 switch (opcode) {
1123 case Op_CallLeaf:
1124 case Op_CallLeafNoFP: {
1125 assert (n->is_Call(), "");
1126 CallNode *call = n->as_Call();
1127 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1128 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
1129 if (call->req() > cnt) {
1130 assert(call->req() == cnt + 1, "only one extra input");
1131 Node *addp = call->in(cnt);
1132 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1133 call->del_req(cnt);
1134 }
1135 }
1136 return false;
1137 }
1138 case Op_ShenandoahCompareAndSwapP:
1139 case Op_ShenandoahCompareAndSwapN:
1140 case Op_ShenandoahWeakCompareAndSwapN:
1141 case Op_ShenandoahWeakCompareAndSwapP:
1142 case Op_ShenandoahCompareAndExchangeP:
1143 case Op_ShenandoahCompareAndExchangeN:
1144 return true;
1145 case Op_ShenandoahLoadReferenceBarrier:
1146 assert(false, "should have been expanded already");
1147 return true;
1148 default:
|
810 Node* ctrl = ac->in(TypeFunc::Control);
811 Node* mem = ac->in(TypeFunc::Memory);
812 Node* src_base = ac->in(ArrayCopyNode::Src);
813 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
814 Node* dest_base = ac->in(ArrayCopyNode::Dest);
815 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
816 Node* length = ac->in(ArrayCopyNode::Length);
817
818 Node* src = phase->basic_plus_adr(src_base, src_offset);
819 Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
820
821 if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
822 // Check if heap is has forwarded objects. If it does, we need to call into the special
823 // routine that would fix up source references before we can continue.
824
825 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
826 Node* region = new RegionNode(PATH_LIMIT);
827 Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
828
829 Node* thread = phase->transform_later(new ThreadLocalNode());
830 Node* offset = phase->MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
831 Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
832
833 uint gc_state_idx = Compile::AliasIdxRaw;
834 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
835 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
836
837 Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
838 int flags = ShenandoahHeap::HAS_FORWARDED;
839 if (ShenandoahIUBarrier) {
840 flags |= ShenandoahHeap::MARKING;
841 }
842 Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags)));
843 Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
844 Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
845
846 IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
847 Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
848 Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
849
850 // Heap is stable, no need to do anything additional
862
863 ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
864 mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
865 region->init_req(_heap_unstable, ctrl);
866 mem_phi->init_req(_heap_unstable, mem);
867
868 // Wire up the actual arraycopy stub now
869 ctrl = phase->transform_later(region);
870 mem = phase->transform_later(mem_phi);
871
872 const char* name = "arraycopy";
873 call = phase->make_leaf_call(ctrl, mem,
874 OptoRuntime::fast_arraycopy_Type(),
875 phase->basictype2arraycopy(T_LONG, NULL, NULL, true, name, true),
876 name, TypeRawPtr::BOTTOM,
877 src, dest, length
878 LP64_ONLY(COMMA phase->top()));
879 call = phase->transform_later(call);
880
881 // Hook up the whole thing into the graph
882 phase->replace_node(ac, call);
883 } else {
884 BarrierSetC2::clone_at_expansion(phase, ac);
885 }
886 }
887
888
889 // Support for macro expanded GC barriers
890 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
891 if (node->Opcode() == Op_ShenandoahIUBarrier) {
892 state()->add_iu_barrier((ShenandoahIUBarrierNode*) node);
893 }
894 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
895 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
896 }
897 }
898
899 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
900 if (node->Opcode() == Op_ShenandoahIUBarrier) {
901 state()->remove_iu_barrier((ShenandoahIUBarrierNode*) node);
902 }
903 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
904 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
905 }
906 }
907
908 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* n) const {
909 if (is_shenandoah_wb_pre_call(n)) {
910 shenandoah_eliminate_wb_pre(n, igvn);
911 }
912 }
913
914 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
915 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
916 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
917 c = c->unique_ctrl_out();
918 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
919 c = c->unique_ctrl_out();
920 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
921 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
922 assert(iff->is_If(), "expect test");
923 if (!is_shenandoah_marking_if(igvn, iff)) {
924 c = c->unique_ctrl_out();
925 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
926 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
927 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
928 }
929 Node* cmpx = iff->in(1)->in(1);
930 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
1023 if (if_ctrl != load_ctrl) {
1024 // Skip possible CProj->NeverBranch in infinite loops
1025 if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1026 && (if_ctrl->in(0)->is_MultiBranch() && if_ctrl->in(0)->Opcode() == Op_NeverBranch)) {
1027 if_ctrl = if_ctrl->in(0)->in(0);
1028 }
1029 }
1030 assert(load_ctrl != NULL && if_ctrl == load_ctrl, "controls must match");
1031 }
1032 }
1033 }
1034 }
1035 }
1036 }
1037 }
1038 }
1039 #endif
1040
1041 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1042 if (is_shenandoah_wb_pre_call(n)) {
1043 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain_sig()->cnt();
1044 if (n->req() > cnt) {
1045 Node* addp = n->in(cnt);
1046 if (has_only_shenandoah_wb_pre_uses(addp)) {
1047 n->del_req(cnt);
1048 if (can_reshape) {
1049 phase->is_IterGVN()->_worklist.push(addp);
1050 }
1051 return n;
1052 }
1053 }
1054 }
1055 if (n->Opcode() == Op_CmpP) {
1056 Node* in1 = n->in(1);
1057 Node* in2 = n->in(2);
1058
1059 // If one input is NULL, then step over the strong LRB barriers on the other input
1060 if (in1->bottom_type() == TypePtr::NULL_PTR &&
1061 !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1062 !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1063 in2 = step_over_gc_barrier(in2);
1108 return NULL;
1109 }
1110
1111 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1112 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1113 Node* u = n->fast_out(i);
1114 if (!is_shenandoah_wb_pre_call(u)) {
1115 return false;
1116 }
1117 }
1118 return n->outcnt() > 0;
1119 }
1120
1121 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
1122 switch (opcode) {
1123 case Op_CallLeaf:
1124 case Op_CallLeafNoFP: {
1125 assert (n->is_Call(), "");
1126 CallNode *call = n->as_Call();
1127 if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1128 uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain_sig()->cnt();
1129 if (call->req() > cnt) {
1130 assert(call->req() == cnt + 1, "only one extra input");
1131 Node *addp = call->in(cnt);
1132 assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1133 call->del_req(cnt);
1134 }
1135 }
1136 return false;
1137 }
1138 case Op_ShenandoahCompareAndSwapP:
1139 case Op_ShenandoahCompareAndSwapN:
1140 case Op_ShenandoahWeakCompareAndSwapN:
1141 case Op_ShenandoahWeakCompareAndSwapP:
1142 case Op_ShenandoahCompareAndExchangeP:
1143 case Op_ShenandoahCompareAndExchangeN:
1144 return true;
1145 case Op_ShenandoahLoadReferenceBarrier:
1146 assert(false, "should have been expanded already");
1147 return true;
1148 default:
|