< prev index next >

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

Print this page




 999   phase->register_new_node(in_cset_fast_test_adr, ctrl);
1000   uint in_cset_fast_test_idx = Compile::AliasIdxRaw;
1001   const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument
1002   debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx));
1003   Node* in_cset_fast_test_load = new LoadBNode(ctrl, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BYTE, MemNode::unordered);
1004   phase->register_new_node(in_cset_fast_test_load, ctrl);
1005   Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT));
1006   phase->register_new_node(in_cset_fast_test_cmp, ctrl);
1007   Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::eq);
1008   phase->register_new_node(in_cset_fast_test_test, ctrl);
1009   IfNode* in_cset_fast_test_iff = new IfNode(ctrl, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
1010   phase->register_control(in_cset_fast_test_iff, loop, ctrl);
1011 
1012   not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff);
1013   phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff);
1014 
1015   ctrl = new IfFalseNode(in_cset_fast_test_iff);
1016   phase->register_control(ctrl, loop, in_cset_fast_test_iff);
1017 }
1018 
1019 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) {
1020   IdealLoopTree*loop = phase->get_loop(ctrl);
1021   const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
1022 
1023   // The slow path stub consumes and produces raw memory in addition
1024   // to the existing memory edges
1025   Node* base = find_bottom_mem(ctrl, phase);
1026   MergeMemNode* mm = MergeMemNode::make(base);
1027   mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
1028   phase->register_new_node(mm, ctrl);
1029 




1030   address calladdr = is_native ? CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native)
1031                                : CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier);
1032   const char* name = is_native ? "oop_load_from_native_barrier" : "load_reference_barrier";
1033   Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM);

1034   call->init_req(TypeFunc::Control, ctrl);
1035   call->init_req(TypeFunc::I_O, phase->C->top());
1036   call->init_req(TypeFunc::Memory, mm);
1037   call->init_req(TypeFunc::FramePtr, phase->C->top());
1038   call->init_req(TypeFunc::ReturnAdr, phase->C->top());
1039   call->init_req(TypeFunc::Parms, val);

1040   phase->register_control(call, loop, ctrl);
1041   ctrl = new ProjNode(call, TypeFunc::Control);
1042   phase->register_control(ctrl, loop, call);
1043   result_mem = new ProjNode(call, TypeFunc::Memory);
1044   phase->register_new_node(result_mem, call);
1045   val = new ProjNode(call, TypeFunc::Parms);
1046   phase->register_new_node(val, call);
1047   val = new CheckCastPPNode(ctrl, val, obj_type);
1048   phase->register_new_node(val, ctrl);
1049 }
1050 
1051 void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
1052   Node* ctrl = phase->get_ctrl(barrier);
1053   Node* init_raw_mem = fixer.find_mem(ctrl, barrier);
1054 
1055   // Update the control of all nodes that should be after the
1056   // barrier control flow
1057   uses.clear();
1058   // Every node that is control dependent on the barrier's input
1059   // control will be after the expanded barrier. The raw memory (if


1384     Node* unc_ctrl = NULL;
1385     if (unc != NULL) {
1386       if (val->in(ShenandoahLoadReferenceBarrierNode::Control) != ctrl) {
1387         unc = NULL;
1388       } else {
1389         unc_ctrl = val->in(ShenandoahLoadReferenceBarrierNode::Control);
1390       }
1391     }
1392 
1393     Node* uncasted_val = val;
1394     if (unc != NULL) {
1395       uncasted_val = val->in(1);
1396     }
1397 
1398     Node* heap_stable_ctrl = NULL;
1399     Node* null_ctrl = NULL;
1400 
1401     assert(val->bottom_type()->make_oopptr(), "need oop");
1402     assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
1403 
1404     enum { _heap_stable = 1, _not_cset, _fwded, _evac_path, _null_path, PATH_LIMIT };
1405     Node* region = new RegionNode(PATH_LIMIT);
1406     Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
1407     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1408 
1409     // Stable path.
1410     test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
1411     IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
1412 
1413     // Heap stable case
1414     region->init_req(_heap_stable, heap_stable_ctrl);
1415     val_phi->init_req(_heap_stable, uncasted_val);
1416     raw_mem_phi->init_req(_heap_stable, raw_mem);
1417 
1418     Node* reg2_ctrl = NULL;
1419     // Null case
1420     test_null(ctrl, val, null_ctrl, phase);
1421     if (null_ctrl != NULL) {
1422       reg2_ctrl = null_ctrl->in(0);
1423       region->init_req(_null_path, null_ctrl);
1424       val_phi->init_req(_null_path, uncasted_val);


1434     Node* not_cset_ctrl = NULL;
1435     in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase);
1436     if (not_cset_ctrl != NULL) {
1437       if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0);
1438       region->init_req(_not_cset, not_cset_ctrl);
1439       val_phi->init_req(_not_cset, uncasted_val);
1440       raw_mem_phi->init_req(_not_cset, raw_mem);
1441     }
1442 
1443     // Resolve object when orig-value is in cset.
1444     // Make the unconditional resolve for fwdptr.
1445     Node* new_val = uncasted_val;
1446     if (unc_ctrl != NULL) {
1447       // Clone the null check in this branch to allow implicit null check
1448       new_val = clone_null_check(ctrl, val, unc_ctrl, phase);
1449       fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
1450 
1451       IfNode* iff = unc_ctrl->in(0)->as_If();
1452       phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
1453     }
1454     Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(oopDesc::mark_offset_in_bytes()));
1455     phase->register_new_node(addr, ctrl);
1456     assert(new_val->bottom_type()->isa_oopptr(), "what else?");
1457     Node* markword = new LoadXNode(ctrl, raw_mem, addr, TypeRawPtr::BOTTOM, TypeX_X, MemNode::unordered);
1458     phase->register_new_node(markword, ctrl);
1459 
1460     // Test if object is forwarded. This is the case if lowest two bits are set.
1461     Node* masked = new AndXNode(markword, phase->igvn().MakeConX(markWord::lock_mask_in_place));
1462     phase->register_new_node(masked, ctrl);
1463     Node* cmp = new CmpXNode(masked, phase->igvn().MakeConX(markWord::marked_value));
1464     phase->register_new_node(cmp, ctrl);
1465 
1466     // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr
1467     Node* bol = new BoolNode(cmp, BoolTest::eq); // Equals 3 means it's forwarded
1468     phase->register_new_node(bol, ctrl);
1469 
1470     IfNode* iff = new IfNode(ctrl, bol, PROB_LIKELY(0.999), COUNT_UNKNOWN);
1471     phase->register_control(iff, loop, ctrl);
1472     Node* if_fwd = new IfTrueNode(iff);
1473     phase->register_control(if_fwd, loop, iff);
1474     Node* if_not_fwd = new IfFalseNode(iff);
1475     phase->register_control(if_not_fwd, loop, iff);
1476 
1477     // Decode forward pointer: since we already have the lowest bits, we can just subtract them
1478     // from the mark word without the need for large immediate mask.
1479     Node* masked2 = new SubXNode(markword, masked);
1480     phase->register_new_node(masked2, if_fwd);
1481     Node* fwdraw = new CastX2PNode(masked2);
1482     fwdraw->init_req(0, if_fwd);
1483     phase->register_new_node(fwdraw, if_fwd);
1484     Node* fwd = new CheckCastPPNode(NULL, fwdraw, val->bottom_type());
1485     phase->register_new_node(fwd, if_fwd);
1486 
1487     // Wire up not-equal-path in slots 3.
1488     region->init_req(_fwded, if_fwd);
1489     val_phi->init_req(_fwded, fwd);
1490     raw_mem_phi->init_req(_fwded, raw_mem);
1491 
1492     // Call lrb-stub and wire up that path in slots 4
1493     Node* result_mem = NULL;
1494     ctrl = if_not_fwd;
1495     fwd = new_val;
1496     call_lrb_stub(ctrl, fwd, result_mem, raw_mem, lrb->is_native(), phase);
































1497     region->init_req(_evac_path, ctrl);
1498     val_phi->init_req(_evac_path, fwd);
1499     raw_mem_phi->init_req(_evac_path, result_mem);
1500 
1501     phase->register_control(region, loop, heap_stable_iff);
1502     Node* out_val = val_phi;
1503     phase->register_new_node(val_phi, region);
1504     phase->register_new_node(raw_mem_phi, region);
1505 
1506     fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
1507 
1508     ctrl = orig_ctrl;
1509 
1510     if (unc != NULL) {
1511       for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
1512         Node* u = val->fast_out(i);
1513         Node* c = phase->ctrl_or_self(u);
1514         if (u != lrb && (c != ctrl || is_dominator_same_ctrl(c, lrb, u, phase))) {
1515           phase->igvn().rehash_node_delayed(u);
1516           int nb = u->replace_edge(val, out_val);


1676 
1677     region->init_req(_heap_unstable, region2);
1678     phi->init_req(_heap_unstable, phi2);
1679 
1680     phase->register_control(region, loop, heap_stable_ctrl->in(0));
1681     phase->register_new_node(phi, region);
1682 
1683     fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase);
1684     for(uint next = 0; next < uses.size(); next++ ) {
1685       Node *n = uses.at(next);
1686       assert(phase->get_ctrl(n) == init_ctrl, "bad control");
1687       assert(n != init_raw_mem, "should leave input raw mem above the barrier");
1688       phase->set_ctrl(n, region);
1689       follow_barrier_uses(n, init_ctrl, uses, phase);
1690     }
1691     fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
1692 
1693     phase->igvn().replace_node(barrier, pre_val);
1694   }
1695   assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced");




































































1696 
1697 }
1698 
1699 void ShenandoahBarrierC2Support::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
1700   IdealLoopTree *loop = phase->get_loop(iff);
1701   Node* loop_head = loop->_head;
1702   Node* entry_c = loop_head->in(LoopNode::EntryControl);
1703 
1704   Node* bol = iff->in(1);
1705   Node* cmp = bol->in(1);
1706   Node* andi = cmp->in(1);
1707   Node* load = andi->in(1);
1708 
1709   assert(is_gc_state_load(load), "broken");
1710   if (!phase->is_dominator(load->in(0), entry_c)) {
1711     Node* mem_ctrl = NULL;
1712     Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase);
1713     load = load->clone();
1714     load->set_req(MemNode::Memory, mem);
1715     load->set_req(0, entry_c);




 999   phase->register_new_node(in_cset_fast_test_adr, ctrl);
1000   uint in_cset_fast_test_idx = Compile::AliasIdxRaw;
1001   const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument
1002   debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx));
1003   Node* in_cset_fast_test_load = new LoadBNode(ctrl, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BYTE, MemNode::unordered);
1004   phase->register_new_node(in_cset_fast_test_load, ctrl);
1005   Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT));
1006   phase->register_new_node(in_cset_fast_test_cmp, ctrl);
1007   Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::eq);
1008   phase->register_new_node(in_cset_fast_test_test, ctrl);
1009   IfNode* in_cset_fast_test_iff = new IfNode(ctrl, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
1010   phase->register_control(in_cset_fast_test_iff, loop, ctrl);
1011 
1012   not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff);
1013   phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff);
1014 
1015   ctrl = new IfFalseNode(in_cset_fast_test_iff);
1016   phase->register_control(ctrl, loop, in_cset_fast_test_iff);
1017 }
1018 
1019 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) {
1020   IdealLoopTree*loop = phase->get_loop(ctrl);
1021   const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
1022 
1023   // The slow path stub consumes and produces raw memory in addition
1024   // to the existing memory edges
1025   Node* base = find_bottom_mem(ctrl, phase);
1026   MergeMemNode* mm = MergeMemNode::make(base);
1027   mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
1028   phase->register_new_node(mm, ctrl);
1029 
1030   address target = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ?
1031           CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow) :
1032           CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup);
1033 
1034   address calladdr = is_native ? CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native)
1035                                : target;
1036   const char* name = is_native ? "oop_load_from_native_barrier" : "load_reference_barrier";
1037   Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM);
1038 
1039   call->init_req(TypeFunc::Control, ctrl);
1040   call->init_req(TypeFunc::I_O, phase->C->top());
1041   call->init_req(TypeFunc::Memory, mm);
1042   call->init_req(TypeFunc::FramePtr, phase->C->top());
1043   call->init_req(TypeFunc::ReturnAdr, phase->C->top());
1044   call->init_req(TypeFunc::Parms, val);
1045   call->init_req(TypeFunc::Parms+1, load_addr);
1046   phase->register_control(call, loop, ctrl);
1047   ctrl = new ProjNode(call, TypeFunc::Control);
1048   phase->register_control(ctrl, loop, call);
1049   result_mem = new ProjNode(call, TypeFunc::Memory);
1050   phase->register_new_node(result_mem, call);
1051   val = new ProjNode(call, TypeFunc::Parms);
1052   phase->register_new_node(val, call);
1053   val = new CheckCastPPNode(ctrl, val, obj_type);
1054   phase->register_new_node(val, ctrl);
1055 }
1056 
1057 void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
1058   Node* ctrl = phase->get_ctrl(barrier);
1059   Node* init_raw_mem = fixer.find_mem(ctrl, barrier);
1060 
1061   // Update the control of all nodes that should be after the
1062   // barrier control flow
1063   uses.clear();
1064   // Every node that is control dependent on the barrier's input
1065   // control will be after the expanded barrier. The raw memory (if


1390     Node* unc_ctrl = NULL;
1391     if (unc != NULL) {
1392       if (val->in(ShenandoahLoadReferenceBarrierNode::Control) != ctrl) {
1393         unc = NULL;
1394       } else {
1395         unc_ctrl = val->in(ShenandoahLoadReferenceBarrierNode::Control);
1396       }
1397     }
1398 
1399     Node* uncasted_val = val;
1400     if (unc != NULL) {
1401       uncasted_val = val->in(1);
1402     }
1403 
1404     Node* heap_stable_ctrl = NULL;
1405     Node* null_ctrl = NULL;
1406 
1407     assert(val->bottom_type()->make_oopptr(), "need oop");
1408     assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
1409 
1410     enum { _heap_stable = 1, _not_cset, _evac_path, _null_path, PATH_LIMIT };
1411     Node* region = new RegionNode(PATH_LIMIT);
1412     Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
1413     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1414 
1415     // Stable path.
1416     test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
1417     IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
1418 
1419     // Heap stable case
1420     region->init_req(_heap_stable, heap_stable_ctrl);
1421     val_phi->init_req(_heap_stable, uncasted_val);
1422     raw_mem_phi->init_req(_heap_stable, raw_mem);
1423 
1424     Node* reg2_ctrl = NULL;
1425     // Null case
1426     test_null(ctrl, val, null_ctrl, phase);
1427     if (null_ctrl != NULL) {
1428       reg2_ctrl = null_ctrl->in(0);
1429       region->init_req(_null_path, null_ctrl);
1430       val_phi->init_req(_null_path, uncasted_val);


1440     Node* not_cset_ctrl = NULL;
1441     in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase);
1442     if (not_cset_ctrl != NULL) {
1443       if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0);
1444       region->init_req(_not_cset, not_cset_ctrl);
1445       val_phi->init_req(_not_cset, uncasted_val);
1446       raw_mem_phi->init_req(_not_cset, raw_mem);
1447     }
1448 
1449     // Resolve object when orig-value is in cset.
1450     // Make the unconditional resolve for fwdptr.
1451     Node* new_val = uncasted_val;
1452     if (unc_ctrl != NULL) {
1453       // Clone the null check in this branch to allow implicit null check
1454       new_val = clone_null_check(ctrl, val, unc_ctrl, phase);
1455       fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
1456 
1457       IfNode* iff = unc_ctrl->in(0)->as_If();
1458       phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
1459     }





































1460 
1461     // Call lrb-stub and wire up that path in slots 4
1462     Node* result_mem = NULL;
1463 
1464     Node* fwd = new_val;
1465     Node* addr;
1466     if (ShenandoahSelfFixing) {
1467       VectorSet visited(Thread::current()->resource_area());
1468       addr = get_load_addr(phase, visited, lrb);
1469     } else {
1470       addr = phase->igvn().zerocon(T_OBJECT);
1471     }
1472     if (addr->Opcode() == Op_AddP) {
1473       Node* orig_base = addr->in(AddPNode::Base);
1474       Node* base = new CheckCastPPNode(ctrl, orig_base, orig_base->bottom_type(), true);
1475       phase->register_new_node(base, ctrl);
1476       if (addr->in(AddPNode::Base) == addr->in((AddPNode::Address))) {
1477         // Field access
1478         addr = addr->clone();
1479         addr->set_req(AddPNode::Base, base);
1480         addr->set_req(AddPNode::Address, base);
1481         phase->register_new_node(addr, ctrl);
1482       } else {
1483         Node* addr2 = addr->in(AddPNode::Address);
1484         if (addr2->Opcode() == Op_AddP && addr2->in(AddPNode::Base) == addr2->in(AddPNode::Address) &&
1485               addr2->in(AddPNode::Base) == orig_base) {
1486           addr2 = addr2->clone();
1487           addr2->set_req(AddPNode::Base, base);
1488           addr2->set_req(AddPNode::Address, base);
1489           phase->register_new_node(addr2, ctrl);
1490           addr = addr->clone();
1491           addr->set_req(AddPNode::Base, base);
1492           addr->set_req(AddPNode::Address, addr2);
1493           phase->register_new_node(addr, ctrl);
1494         }
1495       }
1496     }
1497     call_lrb_stub(ctrl, fwd, addr, result_mem, raw_mem, lrb->is_native(), phase);
1498     region->init_req(_evac_path, ctrl);
1499     val_phi->init_req(_evac_path, fwd);
1500     raw_mem_phi->init_req(_evac_path, result_mem);
1501 
1502     phase->register_control(region, loop, heap_stable_iff);
1503     Node* out_val = val_phi;
1504     phase->register_new_node(val_phi, region);
1505     phase->register_new_node(raw_mem_phi, region);
1506 
1507     fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
1508 
1509     ctrl = orig_ctrl;
1510 
1511     if (unc != NULL) {
1512       for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
1513         Node* u = val->fast_out(i);
1514         Node* c = phase->ctrl_or_self(u);
1515         if (u != lrb && (c != ctrl || is_dominator_same_ctrl(c, lrb, u, phase))) {
1516           phase->igvn().rehash_node_delayed(u);
1517           int nb = u->replace_edge(val, out_val);


1677 
1678     region->init_req(_heap_unstable, region2);
1679     phi->init_req(_heap_unstable, phi2);
1680 
1681     phase->register_control(region, loop, heap_stable_ctrl->in(0));
1682     phase->register_new_node(phi, region);
1683 
1684     fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase);
1685     for(uint next = 0; next < uses.size(); next++ ) {
1686       Node *n = uses.at(next);
1687       assert(phase->get_ctrl(n) == init_ctrl, "bad control");
1688       assert(n != init_raw_mem, "should leave input raw mem above the barrier");
1689       phase->set_ctrl(n, region);
1690       follow_barrier_uses(n, init_ctrl, uses, phase);
1691     }
1692     fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
1693 
1694     phase->igvn().replace_node(barrier, pre_val);
1695   }
1696   assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced");
1697 
1698 }
1699 
1700 Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) {
1701   if (visited.test_set(in->_idx)) {
1702     return NULL;
1703   }
1704   switch (in->Opcode()) {
1705     case Op_Proj:
1706       return get_load_addr(phase, visited, in->in(0));
1707     case Op_CastPP:
1708     case Op_CheckCastPP:
1709     case Op_DecodeN:
1710     case Op_EncodeP:
1711       return get_load_addr(phase, visited, in->in(1));
1712     case Op_LoadN:
1713     case Op_LoadP:
1714       return in->in(MemNode::Address);
1715     case Op_CompareAndExchangeN:
1716     case Op_CompareAndExchangeP:
1717     case Op_GetAndSetN:
1718     case Op_GetAndSetP:
1719     case Op_ShenandoahCompareAndExchangeP:
1720     case Op_ShenandoahCompareAndExchangeN:
1721       // Those instructions would just have stored a different
1722       // value into the field. No use to attempt to fix it at this point.
1723       return phase->igvn().zerocon(T_OBJECT);
1724     case Op_CMoveP:
1725     case Op_CMoveN: {
1726       Node* t = get_load_addr(phase, visited, in->in(CMoveNode::IfTrue));
1727       Node* f = get_load_addr(phase, visited, in->in(CMoveNode::IfFalse));
1728       // Handle unambiguous cases: single address reported on both branches.
1729       if (t != NULL && f == NULL) return t;
1730       if (t == NULL && f != NULL) return f;
1731       if (t != NULL && t == f)    return t;
1732       // Ambiguity.
1733       return phase->igvn().zerocon(T_OBJECT);
1734     }
1735     case Op_Phi: {
1736       Node* addr = NULL;
1737       for (uint i = 1; i < in->req(); i++) {
1738         Node* addr1 = get_load_addr(phase, visited, in->in(i));
1739         if (addr == NULL) {
1740           addr = addr1;
1741         }
1742         if (addr != addr1) {
1743           return phase->igvn().zerocon(T_OBJECT);
1744         }
1745       }
1746       return addr;
1747     }
1748     case Op_ShenandoahLoadReferenceBarrier:
1749       return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
1750     case Op_ShenandoahEnqueueBarrier:
1751       return get_load_addr(phase, visited, in->in(1));
1752     case Op_CallDynamicJava:
1753     case Op_CallLeaf:
1754     case Op_CallStaticJava:
1755     case Op_ConN:
1756     case Op_ConP:
1757     case Op_Parm:
1758       return phase->igvn().zerocon(T_OBJECT);
1759     default:
1760 #ifdef ASSERT
1761       fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]);
1762 #endif
1763       return phase->igvn().zerocon(T_OBJECT);
1764   }
1765 
1766 }
1767 
1768 void ShenandoahBarrierC2Support::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
1769   IdealLoopTree *loop = phase->get_loop(iff);
1770   Node* loop_head = loop->_head;
1771   Node* entry_c = loop_head->in(LoopNode::EntryControl);
1772 
1773   Node* bol = iff->in(1);
1774   Node* cmp = bol->in(1);
1775   Node* andi = cmp->in(1);
1776   Node* load = andi->in(1);
1777 
1778   assert(is_gc_state_load(load), "broken");
1779   if (!phase->is_dominator(load->in(0), entry_c)) {
1780     Node* mem_ctrl = NULL;
1781     Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase);
1782     load = load->clone();
1783     load->set_req(MemNode::Memory, mem);
1784     load->set_req(0, entry_c);


< prev index next >