693 ++found;
694 } else if (use->is_CallLeaf()) {
695 CallLeafNode* call = use->as_CallLeaf();
696 if (strcmp(call->_name, "write_ref_field_pre_entry") == 0) {
697 ++found;
698 }
699 }
700 }
701 if (found == 3) {
702 return true;
703 }
704 }
705 }
706 return false;
707 }
708
709 bool G1BarrierSetC2::is_gc_pre_barrier_node(Node *node) const {
710 return is_g1_pre_val_load(node);
711 }
712
713 void G1BarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
714 if (is_g1_pre_val_load(node)) {
715 macro->replace_node(node, macro->zerocon(node->as_Load()->bottom_type()->basic_type()));
716 } else {
717 assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required");
718 assert(node->outcnt() <= 2, "expects 1 or 2 users: Xor and URShift nodes");
719 // It could be only one user, URShift node, in Object.clone() intrinsic
720 // but the new allocation is passed to arraycopy stub and it could not
721 // be scalar replaced. So we don't check the case.
722
723 // An other case of only one user (Xor) is when the value check for null
724 // in G1 post barrier is folded after CCP so the code which used URShift
725 // is removed.
726
727 // Take Region node before eliminating post barrier since it also
728 // eliminates CastP2X node when it has only one user.
729 Node* this_region = node->in(0);
730 assert(this_region != nullptr, "");
731
732 // Remove G1 post barrier.
733
734 // Search for CastP2X->Xor->URShift->Cmp path which
735 // checks if the store done to a different from the value's region.
736 // And replace Cmp with #0 (false) to collapse G1 post barrier.
737 Node* xorx = node->find_out_with(Op_XorX);
738 if (xorx != nullptr) {
739 Node* shift = xorx->unique_out();
740 Node* cmpx = shift->unique_out();
741 assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
742 cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
743 "missing region check in G1 post barrier");
744 macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ));
745
746 // Remove G1 pre barrier.
747
748 // Search "if (marking != 0)" check and set it to "false".
749 // There is no G1 pre barrier if previous stored value is null
750 // (for example, after initialization).
751 if (this_region->is_Region() && this_region->req() == 3) {
752 int ind = 1;
753 if (!this_region->in(ind)->is_IfFalse()) {
754 ind = 2;
755 }
756 if (this_region->in(ind)->is_IfFalse() &&
757 this_region->in(ind)->in(0)->Opcode() == Op_If) {
758 Node* bol = this_region->in(ind)->in(0)->in(1);
759 assert(bol->is_Bool(), "");
760 cmpx = bol->in(1);
761 if (bol->as_Bool()->_test._test == BoolTest::ne &&
762 cmpx->is_Cmp() && cmpx->in(2) == macro->intcon(0) &&
763 cmpx->in(1)->is_Load()) {
764 Node* adr = cmpx->in(1)->as_Load()->in(MemNode::Address);
765 const int marking_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
766 if (adr->is_AddP() && adr->in(AddPNode::Base) == macro->top() &&
767 adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
768 adr->in(AddPNode::Offset) == macro->MakeConX(marking_offset)) {
769 macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ));
770 }
771 }
772 }
773 }
774 } else {
775 assert(!use_ReduceInitialCardMarks(), "can only happen with card marking");
776 // This is a G1 post barrier emitted by the Object.clone() intrinsic.
777 // Search for the CastP2X->URShiftX->AddP->LoadB->Cmp path which checks if the card
778 // is marked as young_gen and replace the Cmp with 0 (false) to collapse the barrier.
779 Node* shift = node->find_out_with(Op_URShiftX);
780 assert(shift != nullptr, "missing G1 post barrier");
781 Node* addp = shift->unique_out();
782 Node* load = addp->find_out_with(Op_LoadB);
783 assert(load != nullptr, "missing G1 post barrier");
784 Node* cmpx = load->unique_out();
785 assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
786 cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
787 "missing card value check in G1 post barrier");
788 macro->replace_node(cmpx, macro->makecon(TypeInt::CC_EQ));
789 // There is no G1 pre barrier in this case
790 }
791 // Now CastP2X can be removed since it is used only on dead path
792 // which currently still alive until igvn optimize it.
793 assert(node->outcnt() == 0 || node->unique_out()->Opcode() == Op_URShiftX, "");
794 macro->replace_node(node, macro->top());
795 }
796 }
797
798 Node* G1BarrierSetC2::step_over_gc_barrier(Node* c) const {
799 if (!use_ReduceInitialCardMarks() &&
800 c != nullptr && c->is_Region() && c->req() == 3) {
801 for (uint i = 1; i < c->req(); i++) {
802 if (c->in(i) != nullptr && c->in(i)->is_Region() &&
803 c->in(i)->req() == 3) {
804 Node* r = c->in(i);
805 for (uint j = 1; j < r->req(); j++) {
806 if (r->in(j) != nullptr && r->in(j)->is_Proj() &&
807 r->in(j)->in(0) != nullptr &&
808 r->in(j)->in(0)->Opcode() == Op_CallLeaf &&
809 r->in(j)->in(0)->as_Call()->entry_point() == CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry)) {
810 Node* call = r->in(j)->in(0);
811 c = c->in(i == 1 ? 2 : 1);
812 if (c != nullptr && c->Opcode() != Op_Parm) {
813 c = c->in(0);
814 if (c != nullptr) {
|
693 ++found;
694 } else if (use->is_CallLeaf()) {
695 CallLeafNode* call = use->as_CallLeaf();
696 if (strcmp(call->_name, "write_ref_field_pre_entry") == 0) {
697 ++found;
698 }
699 }
700 }
701 if (found == 3) {
702 return true;
703 }
704 }
705 }
706 return false;
707 }
708
709 bool G1BarrierSetC2::is_gc_pre_barrier_node(Node *node) const {
710 return is_g1_pre_val_load(node);
711 }
712
713 void G1BarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
714 if (is_g1_pre_val_load(node)) {
715 igvn->replace_node(node, igvn->zerocon(node->as_Load()->bottom_type()->basic_type()));
716 } else {
717 assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required");
718 assert(node->outcnt() <= 2, "expects 1 or 2 users: Xor and URShift nodes");
719 // It could be only one user, URShift node, in Object.clone() intrinsic
720 // but the new allocation is passed to arraycopy stub and it could not
721 // be scalar replaced. So we don't check the case.
722
723 // An other case of only one user (Xor) is when the value check for null
724 // in G1 post barrier is folded after CCP so the code which used URShift
725 // is removed.
726
727 // Take Region node before eliminating post barrier since it also
728 // eliminates CastP2X node when it has only one user.
729 Node* this_region = node->in(0);
730 assert(this_region != nullptr, "");
731
732 // Remove G1 post barrier.
733
734 // Search for CastP2X->Xor->URShift->Cmp path which
735 // checks if the store done to a different from the value's region.
736 // And replace Cmp with #0 (false) to collapse G1 post barrier.
737 Node* xorx = node->find_out_with(Op_XorX);
738 if (xorx != nullptr) {
739 Node* shift = xorx->unique_out();
740 Node* cmpx = shift->unique_out();
741 assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
742 cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
743 "missing region check in G1 post barrier");
744 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
745
746 // Remove G1 pre barrier.
747
748 // Search "if (marking != 0)" check and set it to "false".
749 // There is no G1 pre barrier if previous stored value is null
750 // (for example, after initialization).
751 if (this_region->is_Region() && this_region->req() == 3) {
752 for (int i = 1; i < 3; ++i) {
753 if (this_region->in(i)->is_IfFalse() &&
754 this_region->in(i)->in(0)->is_If() &&
755 this_region->in(i)->in(0)->in(1)->is_Bool()) {
756 Node* bol = this_region->in(i)->in(0)->in(1);
757 cmpx = bol->in(1);
758 if (bol->as_Bool()->_test._test == BoolTest::ne &&
759 cmpx->is_Cmp() && cmpx->in(2) == igvn->intcon(0) &&
760 cmpx->in(1)->is_Load()) {
761 Node* adr = cmpx->in(1)->as_Load()->in(MemNode::Address);
762 const int marking_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
763 if (adr->is_AddP() && adr->in(AddPNode::Base) == igvn->C->top() &&
764 adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
765 adr->in(AddPNode::Offset) == igvn->MakeConX(marking_offset)) {
766 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
767 }
768 }
769 }
770 }
771 }
772 } else {
773 assert(!use_ReduceInitialCardMarks(), "can only happen with card marking");
774 // This is a G1 post barrier emitted by the Object.clone() intrinsic.
775 // Search for the CastP2X->URShiftX->AddP->LoadB->Cmp path which checks if the card
776 // is marked as young_gen and replace the Cmp with 0 (false) to collapse the barrier.
777 Node* shift = node->find_out_with(Op_URShiftX);
778 assert(shift != nullptr, "missing G1 post barrier");
779 Node* addp = shift->unique_out();
780 Node* load = addp->find_out_with(Op_LoadB);
781 assert(load != nullptr, "missing G1 post barrier");
782 Node* cmpx = load->unique_out();
783 assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
784 cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
785 "missing card value check in G1 post barrier");
786 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
787 // There is no G1 pre barrier in this case
788 }
789 // Now CastP2X can be removed since it is used only on dead path
790 // which currently still alive until igvn optimize it.
791 assert(node->outcnt() == 0 || node->unique_out()->Opcode() == Op_URShiftX, "");
792 igvn->replace_node(node, igvn->C->top());
793 }
794 }
795
796 Node* G1BarrierSetC2::step_over_gc_barrier(Node* c) const {
797 if (!use_ReduceInitialCardMarks() &&
798 c != nullptr && c->is_Region() && c->req() == 3) {
799 for (uint i = 1; i < c->req(); i++) {
800 if (c->in(i) != nullptr && c->in(i)->is_Region() &&
801 c->in(i)->req() == 3) {
802 Node* r = c->in(i);
803 for (uint j = 1; j < r->req(); j++) {
804 if (r->in(j) != nullptr && r->in(j)->is_Proj() &&
805 r->in(j)->in(0) != nullptr &&
806 r->in(j)->in(0)->Opcode() == Op_CallLeaf &&
807 r->in(j)->in(0)->as_Call()->entry_point() == CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry)) {
808 Node* call = r->in(j)->in(0);
809 c = c->in(i == 1 ? 2 : 1);
810 if (c != nullptr && c->Opcode() != Op_Parm) {
811 c = c->in(0);
812 if (c != nullptr) {
|