< prev index next >

src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp

Print this page

 810   Node* ctrl = ac->in(TypeFunc::Control);
 811   Node* mem = ac->in(TypeFunc::Memory);
 812   Node* src_base = ac->in(ArrayCopyNode::Src);
 813   Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
 814   Node* dest_base = ac->in(ArrayCopyNode::Dest);
 815   Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
 816   Node* length = ac->in(ArrayCopyNode::Length);
 817 
 818   Node* src = phase->basic_plus_adr(src_base, src_offset);
 819   Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
 820 
 821   if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
 822     // Check if heap is has forwarded objects. If it does, we need to call into the special
 823     // routine that would fix up source references before we can continue.
 824 
 825     enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
 826     Node* region = new RegionNode(PATH_LIMIT);
 827     Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
 828 
 829     Node* thread = phase->transform_later(new ThreadLocalNode());
 830     Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 831     Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
 832 
 833     uint gc_state_idx = Compile::AliasIdxRaw;
 834     const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
 835     debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
 836 
 837     Node* gc_state    = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
 838     int flags = ShenandoahHeap::HAS_FORWARDED;
 839     if (ShenandoahIUBarrier) {
 840       flags |= ShenandoahHeap::MARKING;
 841     }
 842     Node* stable_and  = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags)));
 843     Node* stable_cmp  = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
 844     Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
 845 
 846     IfNode* stable_iff  = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
 847     Node* stable_ctrl   = phase->transform_later(new IfFalseNode(stable_iff));
 848     Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
 849 
 850     // Heap is stable, no need to do anything additional

 862 
 863     ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
 864     mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
 865     region->init_req(_heap_unstable, ctrl);
 866     mem_phi->init_req(_heap_unstable, mem);
 867 
 868     // Wire up the actual arraycopy stub now
 869     ctrl = phase->transform_later(region);
 870     mem = phase->transform_later(mem_phi);
 871 
 872     const char* name = "arraycopy";
 873     call = phase->make_leaf_call(ctrl, mem,
 874                                  OptoRuntime::fast_arraycopy_Type(),
 875                                  phase->basictype2arraycopy(T_LONG, NULL, NULL, true, name, true),
 876                                  name, TypeRawPtr::BOTTOM,
 877                                  src, dest, length
 878                                  LP64_ONLY(COMMA phase->top()));
 879     call = phase->transform_later(call);
 880 
 881     // Hook up the whole thing into the graph
 882     phase->igvn().replace_node(ac, call);
 883   } else {
 884     BarrierSetC2::clone_at_expansion(phase, ac);
 885   }
 886 }
 887 
 888 
 889 // Support for macro expanded GC barriers
 890 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
 891   if (node->Opcode() == Op_ShenandoahIUBarrier) {
 892     state()->add_iu_barrier((ShenandoahIUBarrierNode*) node);
 893   }
 894   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
 895     state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
 896   }
 897 }
 898 
 899 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
 900   if (node->Opcode() == Op_ShenandoahIUBarrier) {
 901     state()->remove_iu_barrier((ShenandoahIUBarrierNode*) node);
 902   }
 903   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
 904     state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
 905   }
 906 }
 907 
 908 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
 909   if (is_shenandoah_wb_pre_call(n)) {
 910     shenandoah_eliminate_wb_pre(n, &macro->igvn());
 911   }
 912 }
 913 
 914 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
 915   assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
 916   Node* c = call->as_Call()->proj_out(TypeFunc::Control);
 917   c = c->unique_ctrl_out();
 918   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 919   c = c->unique_ctrl_out();
 920   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 921   Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
 922   assert(iff->is_If(), "expect test");
 923   if (!is_shenandoah_marking_if(igvn, iff)) {
 924     c = c->unique_ctrl_out();
 925     assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 926     iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
 927     assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
 928   }
 929   Node* cmpx = iff->in(1)->in(1);
 930   igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));

1023                 if (if_ctrl != load_ctrl) {
1024                   // Skip possible CProj->NeverBranch in infinite loops
1025                   if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1026                       && (if_ctrl->in(0)->is_MultiBranch() && if_ctrl->in(0)->Opcode() == Op_NeverBranch)) {
1027                     if_ctrl = if_ctrl->in(0)->in(0);
1028                   }
1029                 }
1030                 assert(load_ctrl != NULL && if_ctrl == load_ctrl, "controls must match");
1031               }
1032             }
1033           }
1034         }
1035       }
1036     }
1037   }
1038 }
1039 #endif
1040 
1041 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1042   if (is_shenandoah_wb_pre_call(n)) {
1043     uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
1044     if (n->req() > cnt) {
1045       Node* addp = n->in(cnt);
1046       if (has_only_shenandoah_wb_pre_uses(addp)) {
1047         n->del_req(cnt);
1048         if (can_reshape) {
1049           phase->is_IterGVN()->_worklist.push(addp);
1050         }
1051         return n;
1052       }
1053     }
1054   }
1055   if (n->Opcode() == Op_CmpP) {
1056     Node* in1 = n->in(1);
1057     Node* in2 = n->in(2);
1058 
1059     // If one input is NULL, then step over the strong LRB barriers on the other input
1060     if (in1->bottom_type() == TypePtr::NULL_PTR &&
1061         !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1062           !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1063       in2 = step_over_gc_barrier(in2);

1108   return NULL;
1109 }
1110 
1111 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1112   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1113     Node* u = n->fast_out(i);
1114     if (!is_shenandoah_wb_pre_call(u)) {
1115       return false;
1116     }
1117   }
1118   return n->outcnt() > 0;
1119 }
1120 
1121 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
1122   switch (opcode) {
1123     case Op_CallLeaf:
1124     case Op_CallLeafNoFP: {
1125       assert (n->is_Call(), "");
1126       CallNode *call = n->as_Call();
1127       if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1128         uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain()->cnt();
1129         if (call->req() > cnt) {
1130           assert(call->req() == cnt + 1, "only one extra input");
1131           Node *addp = call->in(cnt);
1132           assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1133           call->del_req(cnt);
1134         }
1135       }
1136       return false;
1137     }
1138     case Op_ShenandoahCompareAndSwapP:
1139     case Op_ShenandoahCompareAndSwapN:
1140     case Op_ShenandoahWeakCompareAndSwapN:
1141     case Op_ShenandoahWeakCompareAndSwapP:
1142     case Op_ShenandoahCompareAndExchangeP:
1143     case Op_ShenandoahCompareAndExchangeN:
1144       return true;
1145     case Op_ShenandoahLoadReferenceBarrier:
1146       assert(false, "should have been expanded already");
1147       return true;
1148     default:

 810   Node* ctrl = ac->in(TypeFunc::Control);
 811   Node* mem = ac->in(TypeFunc::Memory);
 812   Node* src_base = ac->in(ArrayCopyNode::Src);
 813   Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
 814   Node* dest_base = ac->in(ArrayCopyNode::Dest);
 815   Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
 816   Node* length = ac->in(ArrayCopyNode::Length);
 817 
 818   Node* src = phase->basic_plus_adr(src_base, src_offset);
 819   Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
 820 
 821   if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
 822     // Check if heap is has forwarded objects. If it does, we need to call into the special
 823     // routine that would fix up source references before we can continue.
 824 
 825     enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
 826     Node* region = new RegionNode(PATH_LIMIT);
 827     Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
 828 
 829     Node* thread = phase->transform_later(new ThreadLocalNode());
 830     Node* offset = phase->MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 831     Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
 832 
 833     uint gc_state_idx = Compile::AliasIdxRaw;
 834     const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
 835     debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
 836 
 837     Node* gc_state    = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
 838     int flags = ShenandoahHeap::HAS_FORWARDED;
 839     if (ShenandoahIUBarrier) {
 840       flags |= ShenandoahHeap::MARKING;
 841     }
 842     Node* stable_and  = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags)));
 843     Node* stable_cmp  = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
 844     Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
 845 
 846     IfNode* stable_iff  = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
 847     Node* stable_ctrl   = phase->transform_later(new IfFalseNode(stable_iff));
 848     Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
 849 
 850     // Heap is stable, no need to do anything additional

 862 
 863     ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
 864     mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
 865     region->init_req(_heap_unstable, ctrl);
 866     mem_phi->init_req(_heap_unstable, mem);
 867 
 868     // Wire up the actual arraycopy stub now
 869     ctrl = phase->transform_later(region);
 870     mem = phase->transform_later(mem_phi);
 871 
 872     const char* name = "arraycopy";
 873     call = phase->make_leaf_call(ctrl, mem,
 874                                  OptoRuntime::fast_arraycopy_Type(),
 875                                  phase->basictype2arraycopy(T_LONG, NULL, NULL, true, name, true),
 876                                  name, TypeRawPtr::BOTTOM,
 877                                  src, dest, length
 878                                  LP64_ONLY(COMMA phase->top()));
 879     call = phase->transform_later(call);
 880 
 881     // Hook up the whole thing into the graph
 882     phase->replace_node(ac, call);
 883   } else {
 884     BarrierSetC2::clone_at_expansion(phase, ac);
 885   }
 886 }
 887 
 888 
 889 // Support for macro expanded GC barriers
 890 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
 891   if (node->Opcode() == Op_ShenandoahIUBarrier) {
 892     state()->add_iu_barrier((ShenandoahIUBarrierNode*) node);
 893   }
 894   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
 895     state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
 896   }
 897 }
 898 
 899 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
 900   if (node->Opcode() == Op_ShenandoahIUBarrier) {
 901     state()->remove_iu_barrier((ShenandoahIUBarrierNode*) node);
 902   }
 903   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
 904     state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
 905   }
 906 }
 907 
 908 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* n) const {
 909   if (is_shenandoah_wb_pre_call(n)) {
 910     shenandoah_eliminate_wb_pre(n, igvn);
 911   }
 912 }
 913 
 914 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
 915   assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
 916   Node* c = call->as_Call()->proj_out(TypeFunc::Control);
 917   c = c->unique_ctrl_out();
 918   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 919   c = c->unique_ctrl_out();
 920   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 921   Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
 922   assert(iff->is_If(), "expect test");
 923   if (!is_shenandoah_marking_if(igvn, iff)) {
 924     c = c->unique_ctrl_out();
 925     assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 926     iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
 927     assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
 928   }
 929   Node* cmpx = iff->in(1)->in(1);
 930   igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));

1023                 if (if_ctrl != load_ctrl) {
1024                   // Skip possible CProj->NeverBranch in infinite loops
1025                   if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1026                       && (if_ctrl->in(0)->is_MultiBranch() && if_ctrl->in(0)->Opcode() == Op_NeverBranch)) {
1027                     if_ctrl = if_ctrl->in(0)->in(0);
1028                   }
1029                 }
1030                 assert(load_ctrl != NULL && if_ctrl == load_ctrl, "controls must match");
1031               }
1032             }
1033           }
1034         }
1035       }
1036     }
1037   }
1038 }
1039 #endif
1040 
1041 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1042   if (is_shenandoah_wb_pre_call(n)) {
1043     uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain_sig()->cnt();
1044     if (n->req() > cnt) {
1045       Node* addp = n->in(cnt);
1046       if (has_only_shenandoah_wb_pre_uses(addp)) {
1047         n->del_req(cnt);
1048         if (can_reshape) {
1049           phase->is_IterGVN()->_worklist.push(addp);
1050         }
1051         return n;
1052       }
1053     }
1054   }
1055   if (n->Opcode() == Op_CmpP) {
1056     Node* in1 = n->in(1);
1057     Node* in2 = n->in(2);
1058 
1059     // If one input is NULL, then step over the strong LRB barriers on the other input
1060     if (in1->bottom_type() == TypePtr::NULL_PTR &&
1061         !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1062           !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1063       in2 = step_over_gc_barrier(in2);

1108   return NULL;
1109 }
1110 
1111 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1112   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1113     Node* u = n->fast_out(i);
1114     if (!is_shenandoah_wb_pre_call(u)) {
1115       return false;
1116     }
1117   }
1118   return n->outcnt() > 0;
1119 }
1120 
1121 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const {
1122   switch (opcode) {
1123     case Op_CallLeaf:
1124     case Op_CallLeafNoFP: {
1125       assert (n->is_Call(), "");
1126       CallNode *call = n->as_Call();
1127       if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1128         uint cnt = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type()->domain_sig()->cnt();
1129         if (call->req() > cnt) {
1130           assert(call->req() == cnt + 1, "only one extra input");
1131           Node *addp = call->in(cnt);
1132           assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1133           call->del_req(cnt);
1134         }
1135       }
1136       return false;
1137     }
1138     case Op_ShenandoahCompareAndSwapP:
1139     case Op_ShenandoahCompareAndSwapN:
1140     case Op_ShenandoahWeakCompareAndSwapN:
1141     case Op_ShenandoahWeakCompareAndSwapP:
1142     case Op_ShenandoahCompareAndExchangeP:
1143     case Op_ShenandoahCompareAndExchangeN:
1144       return true;
1145     case Op_ShenandoahLoadReferenceBarrier:
1146       assert(false, "should have been expanded already");
1147       return true;
1148     default:
< prev index next >