< prev index next >

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

Print this page




 843 void ShenandoahBarrierC2Support::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) {
 844   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
 845     Node* u = n->fast_out(i);
 846     if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) {
 847       uses.push(u);
 848     }
 849   }
 850 }
 851 
 852 static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNode* inner, PhaseIdealLoop* phase) {
 853   OuterStripMinedLoopEndNode* le = inner->outer_loop_end();
 854   Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl));
 855   phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl));
 856   Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt);
 857   phase->register_control(new_le, phase->get_loop(le), le->in(0));
 858   phase->lazy_replace(outer, new_outer);
 859   phase->lazy_replace(le, new_le);
 860   inner->clear_strip_mined();
 861 }
 862 
 863 void ShenandoahBarrierC2Support::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
 864                                                   PhaseIdealLoop* phase) {
 865   IdealLoopTree* loop = phase->get_loop(ctrl);
 866   Node* thread = new ThreadLocalNode();
 867   phase->register_new_node(thread, ctrl);
 868   Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 869   phase->set_ctrl(offset, phase->C->root());
 870   Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset);
 871   phase->register_new_node(gc_state_addr, ctrl);
 872   uint gc_state_idx = Compile::AliasIdxRaw;
 873   const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
 874   debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
 875 
 876   Node* gc_state = new LoadBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
 877   phase->register_new_node(gc_state, ctrl);
 878   Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED));
 879   phase->register_new_node(heap_stable_and, ctrl);
 880   Node* heap_stable_cmp = new CmpINode(heap_stable_and, phase->igvn().zerocon(T_INT));
 881   phase->register_new_node(heap_stable_cmp, ctrl);
 882   Node* heap_stable_test = new BoolNode(heap_stable_cmp, BoolTest::ne);
 883   phase->register_new_node(heap_stable_test, ctrl);
 884   IfNode* heap_stable_iff = new IfNode(ctrl, heap_stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
 885   phase->register_control(heap_stable_iff, loop, ctrl);
 886 
 887   heap_stable_ctrl = new IfFalseNode(heap_stable_iff);
 888   phase->register_control(heap_stable_ctrl, loop, heap_stable_iff);
 889   ctrl = new IfTrueNode(heap_stable_iff);
 890   phase->register_control(ctrl, loop, heap_stable_iff);
 891 
 892   assert(is_heap_stable_test(heap_stable_iff), "Should match the shape");
 893 }
 894 
 895 void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) {
 896   const Type* val_t = phase->igvn().type(val);
 897   if (val_t->meet(TypePtr::NULL_PTR) == val_t) {
 898     IdealLoopTree* loop = phase->get_loop(ctrl);
 899     Node* null_cmp = new CmpPNode(val, phase->igvn().zerocon(T_OBJECT));
 900     phase->register_new_node(null_cmp, ctrl);
 901     Node* null_test = new BoolNode(null_cmp, BoolTest::ne);
 902     phase->register_new_node(null_test, ctrl);
 903     IfNode* null_iff = new IfNode(ctrl, null_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
 904     phase->register_control(null_iff, loop, ctrl);
 905     ctrl = new IfTrueNode(null_iff);
 906     phase->register_control(ctrl, loop, null_iff);
 907     null_ctrl = new IfFalseNode(null_iff);
 908     phase->register_control(null_ctrl, loop, null_iff);
 909   }
 910 }
 911 
 912 Node* ShenandoahBarrierC2Support::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) {


1420       }
1421     }
1422 
1423     Node* uncasted_val = val;
1424     if (unc != NULL) {
1425       uncasted_val = val->in(1);
1426     }
1427 
1428     Node* heap_stable_ctrl = NULL;
1429     Node* null_ctrl = NULL;
1430 
1431     assert(val->bottom_type()->make_oopptr(), "need oop");
1432     assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
1433 
1434     enum { _heap_stable = 1, _not_cset, _evac_path, _null_path, PATH_LIMIT };
1435     Node* region = new RegionNode(PATH_LIMIT);
1436     Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
1437     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1438 
1439     // Stable path.
1440     test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
1441     IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
1442 
1443     // Heap stable case
1444     region->init_req(_heap_stable, heap_stable_ctrl);
1445     val_phi->init_req(_heap_stable, uncasted_val);
1446     raw_mem_phi->init_req(_heap_stable, raw_mem);
1447 
1448     Node* reg2_ctrl = NULL;
1449     // Null case
1450     test_null(ctrl, val, null_ctrl, phase);
1451     if (null_ctrl != NULL) {
1452       reg2_ctrl = null_ctrl->in(0);
1453       region->init_req(_null_path, null_ctrl);
1454       val_phi->init_req(_null_path, uncasted_val);
1455       raw_mem_phi->init_req(_null_path, raw_mem);
1456     } else {
1457       region->del_req(_null_path);
1458       val_phi->del_req(_null_path);
1459       raw_mem_phi->del_req(_null_path);
1460     }


1591     }
1592 
1593     Node* init_ctrl = ctrl;
1594     IdealLoopTree* loop = phase->get_loop(ctrl);
1595     Node* raw_mem = fixer.find_mem(ctrl, barrier);
1596     Node* init_raw_mem = raw_mem;
1597     Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
1598     Node* heap_stable_ctrl = NULL;
1599     Node* null_ctrl = NULL;
1600     uint last = phase->C->unique();
1601 
1602     enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
1603     Node* region = new RegionNode(PATH_LIMIT);
1604     Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1605 
1606     enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 };
1607     Node* region2 = new RegionNode(PATH_LIMIT2);
1608     Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1609 
1610     // Stable path.
1611     test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
1612     region->init_req(_heap_stable, heap_stable_ctrl);
1613     phi->init_req(_heap_stable, raw_mem);
1614 
1615     // Null path
1616     Node* reg2_ctrl = NULL;
1617     test_null(ctrl, pre_val, null_ctrl, phase);
1618     if (null_ctrl != NULL) {
1619       reg2_ctrl = null_ctrl->in(0);
1620       region2->init_req(_null_path, null_ctrl);
1621       phi2->init_req(_null_path, raw_mem);
1622     } else {
1623       region2->del_req(_null_path);
1624       phi2->del_req(_null_path);
1625     }
1626 
1627     const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1628     const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1629     Node* thread = new ThreadLocalNode();
1630     phase->register_new_node(thread, ctrl);
1631     Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset));




 843 void ShenandoahBarrierC2Support::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) {
 844   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
 845     Node* u = n->fast_out(i);
 846     if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) {
 847       uses.push(u);
 848     }
 849   }
 850 }
 851 
 852 static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNode* inner, PhaseIdealLoop* phase) {
 853   OuterStripMinedLoopEndNode* le = inner->outer_loop_end();
 854   Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl));
 855   phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl));
 856   Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt);
 857   phase->register_control(new_le, phase->get_loop(le), le->in(0));
 858   phase->lazy_replace(outer, new_outer);
 859   phase->lazy_replace(le, new_le);
 860   inner->clear_strip_mined();
 861 }
 862 
 863 void ShenandoahBarrierC2Support::test_heap_state(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
 864                                                  PhaseIdealLoop* phase, int flags) {
 865   IdealLoopTree* loop = phase->get_loop(ctrl);
 866   Node* thread = new ThreadLocalNode();
 867   phase->register_new_node(thread, ctrl);
 868   Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 869   phase->set_ctrl(offset, phase->C->root());
 870   Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset);
 871   phase->register_new_node(gc_state_addr, ctrl);
 872   uint gc_state_idx = Compile::AliasIdxRaw;
 873   const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
 874   debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
 875 
 876   Node* gc_state = new LoadBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
 877   phase->register_new_node(gc_state, ctrl);
 878   Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(flags));
 879   phase->register_new_node(heap_stable_and, ctrl);
 880   Node* heap_stable_cmp = new CmpINode(heap_stable_and, phase->igvn().zerocon(T_INT));
 881   phase->register_new_node(heap_stable_cmp, ctrl);
 882   Node* heap_stable_test = new BoolNode(heap_stable_cmp, BoolTest::ne);
 883   phase->register_new_node(heap_stable_test, ctrl);
 884   IfNode* heap_stable_iff = new IfNode(ctrl, heap_stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
 885   phase->register_control(heap_stable_iff, loop, ctrl);
 886 
 887   heap_stable_ctrl = new IfFalseNode(heap_stable_iff);
 888   phase->register_control(heap_stable_ctrl, loop, heap_stable_iff);
 889   ctrl = new IfTrueNode(heap_stable_iff);
 890   phase->register_control(ctrl, loop, heap_stable_iff);
 891 
 892   assert(is_heap_state_test(heap_stable_iff, flags), "Should match the shape");
 893 }
 894 
 895 void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) {
 896   const Type* val_t = phase->igvn().type(val);
 897   if (val_t->meet(TypePtr::NULL_PTR) == val_t) {
 898     IdealLoopTree* loop = phase->get_loop(ctrl);
 899     Node* null_cmp = new CmpPNode(val, phase->igvn().zerocon(T_OBJECT));
 900     phase->register_new_node(null_cmp, ctrl);
 901     Node* null_test = new BoolNode(null_cmp, BoolTest::ne);
 902     phase->register_new_node(null_test, ctrl);
 903     IfNode* null_iff = new IfNode(ctrl, null_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
 904     phase->register_control(null_iff, loop, ctrl);
 905     ctrl = new IfTrueNode(null_iff);
 906     phase->register_control(ctrl, loop, null_iff);
 907     null_ctrl = new IfFalseNode(null_iff);
 908     phase->register_control(null_ctrl, loop, null_iff);
 909   }
 910 }
 911 
 912 Node* ShenandoahBarrierC2Support::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) {


1420       }
1421     }
1422 
1423     Node* uncasted_val = val;
1424     if (unc != NULL) {
1425       uncasted_val = val->in(1);
1426     }
1427 
1428     Node* heap_stable_ctrl = NULL;
1429     Node* null_ctrl = NULL;
1430 
1431     assert(val->bottom_type()->make_oopptr(), "need oop");
1432     assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
1433 
1434     enum { _heap_stable = 1, _not_cset, _evac_path, _null_path, PATH_LIMIT };
1435     Node* region = new RegionNode(PATH_LIMIT);
1436     Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
1437     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1438 
1439     // Stable path.
1440     test_heap_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::HAS_FORWARDED);
1441     IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
1442 
1443     // Heap stable case
1444     region->init_req(_heap_stable, heap_stable_ctrl);
1445     val_phi->init_req(_heap_stable, uncasted_val);
1446     raw_mem_phi->init_req(_heap_stable, raw_mem);
1447 
1448     Node* reg2_ctrl = NULL;
1449     // Null case
1450     test_null(ctrl, val, null_ctrl, phase);
1451     if (null_ctrl != NULL) {
1452       reg2_ctrl = null_ctrl->in(0);
1453       region->init_req(_null_path, null_ctrl);
1454       val_phi->init_req(_null_path, uncasted_val);
1455       raw_mem_phi->init_req(_null_path, raw_mem);
1456     } else {
1457       region->del_req(_null_path);
1458       val_phi->del_req(_null_path);
1459       raw_mem_phi->del_req(_null_path);
1460     }


1591     }
1592 
1593     Node* init_ctrl = ctrl;
1594     IdealLoopTree* loop = phase->get_loop(ctrl);
1595     Node* raw_mem = fixer.find_mem(ctrl, barrier);
1596     Node* init_raw_mem = raw_mem;
1597     Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
1598     Node* heap_stable_ctrl = NULL;
1599     Node* null_ctrl = NULL;
1600     uint last = phase->C->unique();
1601 
1602     enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
1603     Node* region = new RegionNode(PATH_LIMIT);
1604     Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1605 
1606     enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 };
1607     Node* region2 = new RegionNode(PATH_LIMIT2);
1608     Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1609 
1610     // Stable path.
1611     test_heap_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
1612     region->init_req(_heap_stable, heap_stable_ctrl);
1613     phi->init_req(_heap_stable, raw_mem);
1614 
1615     // Null path
1616     Node* reg2_ctrl = NULL;
1617     test_null(ctrl, pre_val, null_ctrl, phase);
1618     if (null_ctrl != NULL) {
1619       reg2_ctrl = null_ctrl->in(0);
1620       region2->init_req(_null_path, null_ctrl);
1621       phi2->init_req(_null_path, raw_mem);
1622     } else {
1623       region2->del_req(_null_path);
1624       phi2->del_req(_null_path);
1625     }
1626 
1627     const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1628     const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1629     Node* thread = new ThreadLocalNode();
1630     phase->register_new_node(thread, ctrl);
1631     Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset));


< prev index next >