< prev index next >

src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp

Print this page

 872   Node* ctrl = ac->in(TypeFunc::Control);
 873   Node* mem = ac->in(TypeFunc::Memory);
 874   Node* src_base = ac->in(ArrayCopyNode::Src);
 875   Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
 876   Node* dest_base = ac->in(ArrayCopyNode::Dest);
 877   Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
 878   Node* length = ac->in(ArrayCopyNode::Length);
 879 
 880   Node* src = phase->basic_plus_adr(src_base, src_offset);
 881   Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
 882 
 883   if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
 884     // Check if heap is has forwarded objects. If it does, we need to call into the special
 885     // routine that would fix up source references before we can continue.
 886 
 887     enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
 888     Node* region = new RegionNode(PATH_LIMIT);
 889     Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
 890 
 891     Node* thread = phase->transform_later(new ThreadLocalNode());
 892     Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 893     Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
 894 
 895     uint gc_state_idx = Compile::AliasIdxRaw;
 896     const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
 897     DEBUG_ONLY(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
 898 
 899     Node* gc_state    = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
 900     Node* stable_and  = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
 901     Node* stable_cmp  = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
 902     Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
 903 
 904     IfNode* stable_iff  = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
 905     Node* stable_ctrl   = phase->transform_later(new IfFalseNode(stable_iff));
 906     Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
 907 
 908     // Heap is stable, no need to do anything additional
 909     region->init_req(_heap_stable, stable_ctrl);
 910     mem_phi->init_req(_heap_stable, mem);
 911 
 912     // Heap is unstable, call into clone barrier stub

 920 
 921     ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
 922     mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
 923     region->init_req(_heap_unstable, ctrl);
 924     mem_phi->init_req(_heap_unstable, mem);
 925 
 926     // Wire up the actual arraycopy stub now
 927     ctrl = phase->transform_later(region);
 928     mem = phase->transform_later(mem_phi);
 929 
 930     const char* name = "arraycopy";
 931     call = phase->make_leaf_call(ctrl, mem,
 932                                  OptoRuntime::fast_arraycopy_Type(),
 933                                  phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
 934                                  name, TypeRawPtr::BOTTOM,
 935                                  src, dest, length
 936                                  LP64_ONLY(COMMA phase->top()));
 937     call = phase->transform_later(call);
 938 
 939     // Hook up the whole thing into the graph
 940     phase->igvn().replace_node(ac, call);
 941   } else {
 942     BarrierSetC2::clone_at_expansion(phase, ac);
 943   }
 944 }
 945 
 946 
 947 // Support for macro expanded GC barriers
 948 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
 949   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
 950     state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
 951   }
 952 }
 953 
 954 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
 955   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
 956     state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
 957   }
 958 }
 959 
 960 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
 961   if (is_shenandoah_wb_pre_call(node)) {
 962     shenandoah_eliminate_wb_pre(node, &macro->igvn());
 963   }
 964   if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
 965     Node* shift = node->unique_out();
 966     Node* addp = shift->unique_out();
 967     for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
 968       Node* mem = addp->last_out(j);
 969       if (UseCondCardMark && mem->is_Load()) {
 970         assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
 971         // The load is checking if the card has been written so
 972         // replace it with zero to fold the test.
 973         macro->replace_node(mem, macro->intcon(0));
 974         continue;
 975       }
 976       assert(mem->is_Store(), "store required");
 977       macro->replace_node(mem, mem->in(MemNode::Memory));
 978     }
 979   }
 980 }
 981 
 982 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
 983   assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
 984   Node* c = call->as_Call()->proj_out(TypeFunc::Control);
 985   c = c->unique_ctrl_out();
 986   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 987   c = c->unique_ctrl_out();
 988   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 989   Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
 990   assert(iff->is_If(), "expect test");
 991   if (!is_shenandoah_marking_if(igvn, iff)) {
 992     c = c->unique_ctrl_out();
 993     assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 994     iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
 995     assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
 996   }
 997   Node* cmpx = iff->in(1)->in(1);

1089                 if (if_ctrl != load_ctrl) {
1090                   // Skip possible CProj->NeverBranch in infinite loops
1091                   if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1092                       && if_ctrl->in(0)->is_NeverBranch()) {
1093                     if_ctrl = if_ctrl->in(0)->in(0);
1094                   }
1095                 }
1096                 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
1097               }
1098             }
1099           }
1100         }
1101       }
1102     }
1103   }
1104 }
1105 #endif
1106 
1107 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1108   if (is_shenandoah_wb_pre_call(n)) {
1109     uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain()->cnt();
1110     if (n->req() > cnt) {
1111       Node* addp = n->in(cnt);
1112       if (has_only_shenandoah_wb_pre_uses(addp)) {
1113         n->del_req(cnt);
1114         if (can_reshape) {
1115           phase->is_IterGVN()->_worklist.push(addp);
1116         }
1117         return n;
1118       }
1119     }
1120   }
1121   if (n->Opcode() == Op_CmpP) {
1122     Node* in1 = n->in(1);
1123     Node* in2 = n->in(2);
1124 
1125     // If one input is null, then step over the strong LRB barriers on the other input
1126     if (in1->bottom_type() == TypePtr::NULL_PTR &&
1127         !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1128           !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1129       in2 = step_over_gc_barrier(in2);

1175   return nullptr;
1176 }
1177 
1178 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1179   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1180     Node* u = n->fast_out(i);
1181     if (!is_shenandoah_wb_pre_call(u)) {
1182       return false;
1183     }
1184   }
1185   return n->outcnt() > 0;
1186 }
1187 
1188 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1189   switch (opcode) {
1190     case Op_CallLeaf:
1191     case Op_CallLeafNoFP: {
1192       assert (n->is_Call(), "");
1193       CallNode *call = n->as_Call();
1194       if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1195         uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain()->cnt();
1196         if (call->req() > cnt) {
1197           assert(call->req() == cnt + 1, "only one extra input");
1198           Node *addp = call->in(cnt);
1199           assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1200           call->del_req(cnt);
1201         }
1202       }
1203       return false;
1204     }
1205     case Op_ShenandoahCompareAndSwapP:
1206     case Op_ShenandoahCompareAndSwapN:
1207     case Op_ShenandoahWeakCompareAndSwapN:
1208     case Op_ShenandoahWeakCompareAndSwapP:
1209     case Op_ShenandoahCompareAndExchangeP:
1210     case Op_ShenandoahCompareAndExchangeN:
1211       return true;
1212     case Op_ShenandoahLoadReferenceBarrier:
1213       assert(false, "should have been expanded already");
1214       return true;
1215     default:

 872   Node* ctrl = ac->in(TypeFunc::Control);
 873   Node* mem = ac->in(TypeFunc::Memory);
 874   Node* src_base = ac->in(ArrayCopyNode::Src);
 875   Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
 876   Node* dest_base = ac->in(ArrayCopyNode::Dest);
 877   Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
 878   Node* length = ac->in(ArrayCopyNode::Length);
 879 
 880   Node* src = phase->basic_plus_adr(src_base, src_offset);
 881   Node* dest = phase->basic_plus_adr(dest_base, dest_offset);
 882 
 883   if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
 884     // Check if heap is has forwarded objects. If it does, we need to call into the special
 885     // routine that would fix up source references before we can continue.
 886 
 887     enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
 888     Node* region = new RegionNode(PATH_LIMIT);
 889     Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
 890 
 891     Node* thread = phase->transform_later(new ThreadLocalNode());
 892     Node* offset = phase->MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 893     Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
 894 
 895     uint gc_state_idx = Compile::AliasIdxRaw;
 896     const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
 897     DEBUG_ONLY(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
 898 
 899     Node* gc_state    = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
 900     Node* stable_and  = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
 901     Node* stable_cmp  = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
 902     Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
 903 
 904     IfNode* stable_iff  = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
 905     Node* stable_ctrl   = phase->transform_later(new IfFalseNode(stable_iff));
 906     Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
 907 
 908     // Heap is stable, no need to do anything additional
 909     region->init_req(_heap_stable, stable_ctrl);
 910     mem_phi->init_req(_heap_stable, mem);
 911 
 912     // Heap is unstable, call into clone barrier stub

 920 
 921     ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
 922     mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
 923     region->init_req(_heap_unstable, ctrl);
 924     mem_phi->init_req(_heap_unstable, mem);
 925 
 926     // Wire up the actual arraycopy stub now
 927     ctrl = phase->transform_later(region);
 928     mem = phase->transform_later(mem_phi);
 929 
 930     const char* name = "arraycopy";
 931     call = phase->make_leaf_call(ctrl, mem,
 932                                  OptoRuntime::fast_arraycopy_Type(),
 933                                  phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, name, true),
 934                                  name, TypeRawPtr::BOTTOM,
 935                                  src, dest, length
 936                                  LP64_ONLY(COMMA phase->top()));
 937     call = phase->transform_later(call);
 938 
 939     // Hook up the whole thing into the graph
 940     phase->replace_node(ac, call);
 941   } else {
 942     BarrierSetC2::clone_at_expansion(phase, ac);
 943   }
 944 }
 945 
 946 
 947 // Support for macro expanded GC barriers
 948 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
 949   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
 950     state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
 951   }
 952 }
 953 
 954 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
 955   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
 956     state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
 957   }
 958 }
 959 
 960 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
 961   if (is_shenandoah_wb_pre_call(node)) {
 962     shenandoah_eliminate_wb_pre(node, igvn);
 963   }
 964   if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
 965     Node* shift = node->unique_out();
 966     Node* addp = shift->unique_out();
 967     for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
 968       Node* mem = addp->last_out(j);
 969       if (UseCondCardMark && mem->is_Load()) {
 970         assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
 971         // The load is checking if the card has been written so
 972         // replace it with zero to fold the test.
 973         igvn->replace_node(mem, igvn->intcon(0));
 974         continue;
 975       }
 976       assert(mem->is_Store(), "store required");
 977       igvn->replace_node(mem, mem->in(MemNode::Memory));
 978     }
 979   }
 980 }
 981 
 982 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
 983   assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
 984   Node* c = call->as_Call()->proj_out(TypeFunc::Control);
 985   c = c->unique_ctrl_out();
 986   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 987   c = c->unique_ctrl_out();
 988   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 989   Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
 990   assert(iff->is_If(), "expect test");
 991   if (!is_shenandoah_marking_if(igvn, iff)) {
 992     c = c->unique_ctrl_out();
 993     assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 994     iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
 995     assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
 996   }
 997   Node* cmpx = iff->in(1)->in(1);

1089                 if (if_ctrl != load_ctrl) {
1090                   // Skip possible CProj->NeverBranch in infinite loops
1091                   if ((if_ctrl->is_Proj() && if_ctrl->Opcode() == Op_CProj)
1092                       && if_ctrl->in(0)->is_NeverBranch()) {
1093                     if_ctrl = if_ctrl->in(0)->in(0);
1094                   }
1095                 }
1096                 assert(load_ctrl != nullptr && if_ctrl == load_ctrl, "controls must match");
1097               }
1098             }
1099           }
1100         }
1101       }
1102     }
1103   }
1104 }
1105 #endif
1106 
1107 Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const {
1108   if (is_shenandoah_wb_pre_call(n)) {
1109     uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain_sig()->cnt();
1110     if (n->req() > cnt) {
1111       Node* addp = n->in(cnt);
1112       if (has_only_shenandoah_wb_pre_uses(addp)) {
1113         n->del_req(cnt);
1114         if (can_reshape) {
1115           phase->is_IterGVN()->_worklist.push(addp);
1116         }
1117         return n;
1118       }
1119     }
1120   }
1121   if (n->Opcode() == Op_CmpP) {
1122     Node* in1 = n->in(1);
1123     Node* in2 = n->in(2);
1124 
1125     // If one input is null, then step over the strong LRB barriers on the other input
1126     if (in1->bottom_type() == TypePtr::NULL_PTR &&
1127         !((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
1128           !ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
1129       in2 = step_over_gc_barrier(in2);

1175   return nullptr;
1176 }
1177 
1178 bool ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(Node* n) {
1179   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1180     Node* u = n->fast_out(i);
1181     if (!is_shenandoah_wb_pre_call(u)) {
1182       return false;
1183     }
1184   }
1185   return n->outcnt() > 0;
1186 }
1187 
1188 bool ShenandoahBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const {
1189   switch (opcode) {
1190     case Op_CallLeaf:
1191     case Op_CallLeafNoFP: {
1192       assert (n->is_Call(), "");
1193       CallNode *call = n->as_Call();
1194       if (ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(call)) {
1195         uint cnt = ShenandoahBarrierSetC2::write_barrier_pre_Type()->domain_sig()->cnt();
1196         if (call->req() > cnt) {
1197           assert(call->req() == cnt + 1, "only one extra input");
1198           Node *addp = call->in(cnt);
1199           assert(!ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(addp), "useless address computation?");
1200           call->del_req(cnt);
1201         }
1202       }
1203       return false;
1204     }
1205     case Op_ShenandoahCompareAndSwapP:
1206     case Op_ShenandoahCompareAndSwapN:
1207     case Op_ShenandoahWeakCompareAndSwapN:
1208     case Op_ShenandoahWeakCompareAndSwapP:
1209     case Op_ShenandoahCompareAndExchangeP:
1210     case Op_ShenandoahCompareAndExchangeN:
1211       return true;
1212     case Op_ShenandoahLoadReferenceBarrier:
1213       assert(false, "should have been expanded already");
1214       return true;
1215     default:
< prev index next >