< prev index next >

src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp

Print this page

191 
192   if (do_load) {
193     // We need to generate the load of the previous value
194     assert(obj != NULL, "must have a base");
195     assert(adr != NULL, "where are loading from?");
196     assert(pre_val == NULL, "loaded already?");
197     assert(val_type != NULL, "need a type");
198 
199     if (use_ReduceInitialCardMarks()
200         && g1_can_remove_pre_barrier(kit, &kit->gvn(), adr, bt, alias_idx)) {
201       return;
202     }
203 
204   } else {
205     // In this case both val_type and alias_idx are unused.
206     assert(pre_val != NULL, "must be loaded already");
207     // Nothing to be done if pre_val is null.
208     if (pre_val->bottom_type() == TypePtr::NULL_PTR) return;
209     assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here");
210   }
211   assert(bt == T_OBJECT, "or we shouldn't be here");
212 
213   IdealKit ideal(kit, true);
214 
215   Node* tls = __ thread(); // ThreadLocalStorage
216 
217   Node* no_base = __ top();
218   Node* zero  = __ ConI(0);
219   Node* zeroX = __ ConX(0);
220 
221   float likely  = PROB_LIKELY(0.999);
222   float unlikely  = PROB_UNLIKELY(0.999);
223 
224   BasicType active_type = in_bytes(SATBMarkQueue::byte_width_of_active()) == 4 ? T_INT : T_BYTE;
225   assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 4 || in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "flag width");
226 
227   // Offsets into the thread
228   const int marking_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
229   const int index_offset   = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset());
230   const int buffer_offset  = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset());
231 

647   }
648 
649   return load;
650 }
651 
652 bool G1BarrierSetC2::is_gc_barrier_node(Node* node) const {
653   if (CardTableBarrierSetC2::is_gc_barrier_node(node)) {
654     return true;
655   }
656   if (node->Opcode() != Op_CallLeaf) {
657     return false;
658   }
659   CallLeafNode *call = node->as_CallLeaf();
660   if (call->_name == NULL) {
661     return false;
662   }
663 
664   return strcmp(call->_name, "write_ref_field_pre_entry") == 0 || strcmp(call->_name, "write_ref_field_post_entry") == 0;
665 }
666 
667 void G1BarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
668   assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required");
669   assert(node->outcnt() <= 2, "expects 1 or 2 users: Xor and URShift nodes");
670   // It could be only one user, URShift node, in Object.clone() intrinsic
671   // but the new allocation is passed to arraycopy stub and it could not
672   // be scalar replaced. So we don't check the case.
673 
674   // An other case of only one user (Xor) is when the value check for NULL
675   // in G1 post barrier is folded after CCP so the code which used URShift
676   // is removed.
677 
678   // Take Region node before eliminating post barrier since it also
679   // eliminates CastP2X node when it has only one user.
680   Node* this_region = node->in(0);
681   assert(this_region != NULL, "");
682 
683   // Remove G1 post barrier.
684 
685   // Search for CastP2X->Xor->URShift->Cmp path which
686   // checks if the store done to a different from the value's region.
687   // And replace Cmp with #0 (false) to collapse G1 post barrier.
688   Node* xorx = node->find_out_with(Op_XorX);
689   if (xorx != NULL) {
690     Node* shift = xorx->unique_out();
691     Node* cmpx = shift->unique_out();
692     assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
693     cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
694     "missing region check in G1 post barrier");
695     macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ));
696 
697     // Remove G1 pre barrier.
698 
699     // Search "if (marking != 0)" check and set it to "false".
700     // There is no G1 pre barrier if previous stored value is NULL
701     // (for example, after initialization).
702     if (this_region->is_Region() && this_region->req() == 3) {
703       int ind = 1;
704       if (!this_region->in(ind)->is_IfFalse()) {
705         ind = 2;
706       }
707       if (this_region->in(ind)->is_IfFalse() &&
708           this_region->in(ind)->in(0)->Opcode() == Op_If) {
709         Node* bol = this_region->in(ind)->in(0)->in(1);
710         assert(bol->is_Bool(), "");
711         cmpx = bol->in(1);
712         if (bol->as_Bool()->_test._test == BoolTest::ne &&
713             cmpx->is_Cmp() && cmpx->in(2) == macro->intcon(0) &&
714             cmpx->in(1)->is_Load()) {
715           Node* adr = cmpx->in(1)->as_Load()->in(MemNode::Address);
716           const int marking_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
717           if (adr->is_AddP() && adr->in(AddPNode::Base) == macro->top() &&
718               adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
719               adr->in(AddPNode::Offset) == macro->MakeConX(marking_offset)) {
720             macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ));
721           }
722         }
723       }
724     }
725   } else {
726     assert(!use_ReduceInitialCardMarks(), "can only happen with card marking");
727     // This is a G1 post barrier emitted by the Object.clone() intrinsic.
728     // Search for the CastP2X->URShiftX->AddP->LoadB->Cmp path which checks if the card
729     // is marked as young_gen and replace the Cmp with 0 (false) to collapse the barrier.
730     Node* shift = node->find_out_with(Op_URShiftX);
731     assert(shift != NULL, "missing G1 post barrier");
732     Node* addp = shift->unique_out();
733     Node* load = addp->find_out_with(Op_LoadB);
734     assert(load != NULL, "missing G1 post barrier");
735     Node* cmpx = load->unique_out();
736     assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
737            cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
738            "missing card value check in G1 post barrier");
739     macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ));
740     // There is no G1 pre barrier in this case
741   }
742   // Now CastP2X can be removed since it is used only on dead path
743   // which currently still alive until igvn optimize it.
744   assert(node->outcnt() == 0 || node->unique_out()->Opcode() == Op_URShiftX, "");
745   macro->replace_node(node, macro->top());
746 }
747 
748 Node* G1BarrierSetC2::step_over_gc_barrier(Node* c) const {
749   if (!use_ReduceInitialCardMarks() &&
750       c != NULL && c->is_Region() && c->req() == 3) {
751     for (uint i = 1; i < c->req(); i++) {
752       if (c->in(i) != NULL && c->in(i)->is_Region() &&
753           c->in(i)->req() == 3) {
754         Node* r = c->in(i);
755         for (uint j = 1; j < r->req(); j++) {
756           if (r->in(j) != NULL && r->in(j)->is_Proj() &&
757               r->in(j)->in(0) != NULL &&
758               r->in(j)->in(0)->Opcode() == Op_CallLeaf &&
759               r->in(j)->in(0)->as_Call()->entry_point() == CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry)) {
760             Node* call = r->in(j)->in(0);
761             c = c->in(i == 1 ? 2 : 1);
762             if (c != NULL && c->Opcode() != Op_Parm) {
763               c = c->in(0);
764               if (c != NULL) {
765                 c = c->in(0);

191 
192   if (do_load) {
193     // We need to generate the load of the previous value
194     assert(obj != NULL, "must have a base");
195     assert(adr != NULL, "where are loading from?");
196     assert(pre_val == NULL, "loaded already?");
197     assert(val_type != NULL, "need a type");
198 
199     if (use_ReduceInitialCardMarks()
200         && g1_can_remove_pre_barrier(kit, &kit->gvn(), adr, bt, alias_idx)) {
201       return;
202     }
203 
204   } else {
205     // In this case both val_type and alias_idx are unused.
206     assert(pre_val != NULL, "must be loaded already");
207     // Nothing to be done if pre_val is null.
208     if (pre_val->bottom_type() == TypePtr::NULL_PTR) return;
209     assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here");
210   }
211   assert(bt == T_OBJECT || bt == T_INLINE_TYPE, "or we shouldn't be here");
212 
213   IdealKit ideal(kit, true);
214 
215   Node* tls = __ thread(); // ThreadLocalStorage
216 
217   Node* no_base = __ top();
218   Node* zero  = __ ConI(0);
219   Node* zeroX = __ ConX(0);
220 
221   float likely  = PROB_LIKELY(0.999);
222   float unlikely  = PROB_UNLIKELY(0.999);
223 
224   BasicType active_type = in_bytes(SATBMarkQueue::byte_width_of_active()) == 4 ? T_INT : T_BYTE;
225   assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 4 || in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "flag width");
226 
227   // Offsets into the thread
228   const int marking_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
229   const int index_offset   = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset());
230   const int buffer_offset  = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset());
231 

647   }
648 
649   return load;
650 }
651 
652 bool G1BarrierSetC2::is_gc_barrier_node(Node* node) const {
653   if (CardTableBarrierSetC2::is_gc_barrier_node(node)) {
654     return true;
655   }
656   if (node->Opcode() != Op_CallLeaf) {
657     return false;
658   }
659   CallLeafNode *call = node->as_CallLeaf();
660   if (call->_name == NULL) {
661     return false;
662   }
663 
664   return strcmp(call->_name, "write_ref_field_pre_entry") == 0 || strcmp(call->_name, "write_ref_field_post_entry") == 0;
665 }
666 
667 void G1BarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
668   assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required");
669   assert(node->outcnt() <= 2, "expects 1 or 2 users: Xor and URShift nodes");
670   // It could be only one user, URShift node, in Object.clone() intrinsic
671   // but the new allocation is passed to arraycopy stub and it could not
672   // be scalar replaced. So we don't check the case.
673 
674   // An other case of only one user (Xor) is when the value check for NULL
675   // in G1 post barrier is folded after CCP so the code which used URShift
676   // is removed.
677 
678   // Take Region node before eliminating post barrier since it also
679   // eliminates CastP2X node when it has only one user.
680   Node* this_region = node->in(0);
681   assert(this_region != NULL, "");
682 
683   // Remove G1 post barrier.
684 
685   // Search for CastP2X->Xor->URShift->Cmp path which
686   // checks if the store done to a different from the value's region.
687   // And replace Cmp with #0 (false) to collapse G1 post barrier.
688   Node* xorx = node->find_out_with(Op_XorX);
689   if (xorx != NULL) {
690     Node* shift = xorx->unique_out();
691     Node* cmpx = shift->unique_out();
692     assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
693     cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
694     "missing region check in G1 post barrier");
695     igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
696 
697     // Remove G1 pre barrier.
698 
699     // Search "if (marking != 0)" check and set it to "false".
700     // There is no G1 pre barrier if previous stored value is NULL
701     // (for example, after initialization).
702     if (this_region->is_Region() && this_region->req() == 3) {
703       for (int i = 1; i < 3; ++i) {
704         if (this_region->in(i)->is_IfFalse() &&
705             this_region->in(i)->in(0)->is_If() &&
706             this_region->in(i)->in(0)->in(1)->is_Bool()) {
707           Node* bol = this_region->in(i)->in(0)->in(1);
708           cmpx = bol->in(1);
709           if (bol->as_Bool()->_test._test == BoolTest::ne &&
710               cmpx->is_Cmp() && cmpx->in(2) == igvn->intcon(0) &&
711               cmpx->in(1)->is_Load()) {
712             Node* adr = cmpx->in(1)->as_Load()->in(MemNode::Address);
713             const int marking_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
714             if (adr->is_AddP() && adr->in(AddPNode::Base) == igvn->C->top() &&
715                 adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
716                 adr->in(AddPNode::Offset) == igvn->MakeConX(marking_offset)) {
717               igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
718             }


719           }
720         }
721       }
722     }
723   } else {
724     assert(!use_ReduceInitialCardMarks(), "can only happen with card marking");
725     // This is a G1 post barrier emitted by the Object.clone() intrinsic.
726     // Search for the CastP2X->URShiftX->AddP->LoadB->Cmp path which checks if the card
727     // is marked as young_gen and replace the Cmp with 0 (false) to collapse the barrier.
728     Node* shift = node->find_out_with(Op_URShiftX);
729     assert(shift != NULL, "missing G1 post barrier");
730     Node* addp = shift->unique_out();
731     Node* load = addp->find_out_with(Op_LoadB);
732     assert(load != NULL, "missing G1 post barrier");
733     Node* cmpx = load->unique_out();
734     assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
735            cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
736            "missing card value check in G1 post barrier");
737     igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
738     // There is no G1 pre barrier in this case
739   }
740   // Now CastP2X can be removed since it is used only on dead path
741   // which currently still alive until igvn optimize it.
742   assert(node->outcnt() == 0 || node->unique_out()->Opcode() == Op_URShiftX, "");
743   igvn->replace_node(node, igvn->C->top());
744 }
745 
746 Node* G1BarrierSetC2::step_over_gc_barrier(Node* c) const {
747   if (!use_ReduceInitialCardMarks() &&
748       c != NULL && c->is_Region() && c->req() == 3) {
749     for (uint i = 1; i < c->req(); i++) {
750       if (c->in(i) != NULL && c->in(i)->is_Region() &&
751           c->in(i)->req() == 3) {
752         Node* r = c->in(i);
753         for (uint j = 1; j < r->req(); j++) {
754           if (r->in(j) != NULL && r->in(j)->is_Proj() &&
755               r->in(j)->in(0) != NULL &&
756               r->in(j)->in(0)->Opcode() == Op_CallLeaf &&
757               r->in(j)->in(0)->as_Call()->entry_point() == CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry)) {
758             Node* call = r->in(j)->in(0);
759             c = c->in(i == 1 ? 2 : 1);
760             if (c != NULL && c->Opcode() != Op_Parm) {
761               c = c->in(0);
762               if (c != NULL) {
763                 c = c->in(0);
< prev index next >