< prev index next >

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

Print this page




1065   phase->register_new_node(in_cset_fast_test_adr, ctrl);
1066   uint in_cset_fast_test_idx = Compile::AliasIdxRaw;
1067   const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument
1068   debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx));
1069   Node* in_cset_fast_test_load = new LoadBNode(ctrl, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BYTE, MemNode::unordered);
1070   phase->register_new_node(in_cset_fast_test_load, ctrl);
1071   Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT));
1072   phase->register_new_node(in_cset_fast_test_cmp, ctrl);
1073   Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::eq);
1074   phase->register_new_node(in_cset_fast_test_test, ctrl);
1075   IfNode* in_cset_fast_test_iff = new IfNode(ctrl, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
1076   phase->register_control(in_cset_fast_test_iff, loop, ctrl);
1077 
1078   not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff);
1079   phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff);
1080 
1081   ctrl = new IfFalseNode(in_cset_fast_test_iff);
1082   phase->register_control(ctrl, loop, in_cset_fast_test_iff);
1083 }
1084 
1085 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase) {
1086   IdealLoopTree*loop = phase->get_loop(ctrl);
1087   const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
1088 
1089   // The slow path stub consumes and produces raw memory in addition
1090   // to the existing memory edges
1091   Node* base = find_bottom_mem(ctrl, phase);
1092   MergeMemNode* mm = MergeMemNode::make(base);
1093   mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
1094   phase->register_new_node(mm, ctrl);
1095 
1096   Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT), "shenandoah_load_reference_barrier", TypeRawPtr::BOTTOM);




1097   call->init_req(TypeFunc::Control, ctrl);
1098   call->init_req(TypeFunc::I_O, phase->C->top());
1099   call->init_req(TypeFunc::Memory, mm);
1100   call->init_req(TypeFunc::FramePtr, phase->C->top());
1101   call->init_req(TypeFunc::ReturnAdr, phase->C->top());
1102   call->init_req(TypeFunc::Parms, val);

1103   phase->register_control(call, loop, ctrl);
1104   ctrl = new ProjNode(call, TypeFunc::Control);
1105   phase->register_control(ctrl, loop, call);
1106   result_mem = new ProjNode(call, TypeFunc::Memory);
1107   phase->register_new_node(result_mem, call);
1108   val = new ProjNode(call, TypeFunc::Parms);
1109   phase->register_new_node(val, call);
1110   val = new CheckCastPPNode(ctrl, val, obj_type);
1111   phase->register_new_node(val, ctrl);
1112 }
1113 
1114 void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
1115   Node* ctrl = phase->get_ctrl(barrier);
1116   Node* init_raw_mem = fixer.find_mem(ctrl, barrier);
1117 
1118   // Update the control of all nodes that should be after the
1119   // barrier control flow
1120   uses.clear();
1121   // Every node that is control dependent on the barrier's input
1122   // control will be after the expanded barrier. The raw memory (if


1447     Node* unc_ctrl = NULL;
1448     if (unc != NULL) {
1449       if (val->in(ShenandoahLoadReferenceBarrierNode::Control) != ctrl) {
1450         unc = NULL;
1451       } else {
1452         unc_ctrl = val->in(ShenandoahLoadReferenceBarrierNode::Control);
1453       }
1454     }
1455 
1456     Node* uncasted_val = val;
1457     if (unc != NULL) {
1458       uncasted_val = val->in(1);
1459     }
1460 
1461     Node* heap_stable_ctrl = NULL;
1462     Node* null_ctrl = NULL;
1463 
1464     assert(val->bottom_type()->make_oopptr(), "need oop");
1465     assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
1466 
1467     enum { _heap_stable = 1, _not_cset, _fwded, _evac_path, _null_path, PATH_LIMIT };
1468     Node* region = new RegionNode(PATH_LIMIT);
1469     Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
1470     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1471 
1472     // Stable path.
1473     test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
1474     IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
1475 
1476     // Heap stable case
1477     region->init_req(_heap_stable, heap_stable_ctrl);
1478     val_phi->init_req(_heap_stable, uncasted_val);
1479     raw_mem_phi->init_req(_heap_stable, raw_mem);
1480 
1481     Node* reg2_ctrl = NULL;
1482     // Null case
1483     test_null(ctrl, val, null_ctrl, phase);
1484     if (null_ctrl != NULL) {
1485       reg2_ctrl = null_ctrl->in(0);
1486       region->init_req(_null_path, null_ctrl);
1487       val_phi->init_req(_null_path, uncasted_val);


1497     Node* not_cset_ctrl = NULL;
1498     in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase);
1499     if (not_cset_ctrl != NULL) {
1500       if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0);
1501       region->init_req(_not_cset, not_cset_ctrl);
1502       val_phi->init_req(_not_cset, uncasted_val);
1503       raw_mem_phi->init_req(_not_cset, raw_mem);
1504     }
1505 
1506     // Resolve object when orig-value is in cset.
1507     // Make the unconditional resolve for fwdptr.
1508     Node* new_val = uncasted_val;
1509     if (unc_ctrl != NULL) {
1510       // Clone the null check in this branch to allow implicit null check
1511       new_val = clone_null_check(ctrl, val, unc_ctrl, phase);
1512       fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
1513 
1514       IfNode* iff = unc_ctrl->in(0)->as_If();
1515       phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
1516     }
1517     Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(oopDesc::mark_offset_in_bytes()));
1518     phase->register_new_node(addr, ctrl);
1519     assert(new_val->bottom_type()->isa_oopptr(), "what else?");
1520     Node* markword = new LoadXNode(ctrl, raw_mem, addr, TypeRawPtr::BOTTOM, TypeX_X, MemNode::unordered);
1521     phase->register_new_node(markword, ctrl);
1522 
1523     // Test if object is forwarded. This is the case if lowest two bits are set.
1524     Node* masked = new AndXNode(markword, phase->igvn().MakeConX(markOopDesc::lock_mask_in_place));
1525     phase->register_new_node(masked, ctrl);
1526     Node* cmp = new CmpXNode(masked, phase->igvn().MakeConX(markOopDesc::marked_value));
1527     phase->register_new_node(cmp, ctrl);
1528 
1529     // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr
1530     Node* bol = new BoolNode(cmp, BoolTest::eq); // Equals 3 means it's forwarded
1531     phase->register_new_node(bol, ctrl);
1532 
1533     IfNode* iff = new IfNode(ctrl, bol, PROB_LIKELY(0.999), COUNT_UNKNOWN);
1534     phase->register_control(iff, loop, ctrl);
1535     Node* if_fwd = new IfTrueNode(iff);
1536     phase->register_control(if_fwd, loop, iff);
1537     Node* if_not_fwd = new IfFalseNode(iff);
1538     phase->register_control(if_not_fwd, loop, iff);
1539 
1540     // Decode forward pointer: since we already have the lowest bits, we can just subtract them
1541     // from the mark word without the need for large immediate mask.
1542     Node* masked2 = new SubXNode(markword, masked);
1543     phase->register_new_node(masked2, if_fwd);
1544     Node* fwdraw = new CastX2PNode(masked2);
1545     fwdraw->init_req(0, if_fwd);
1546     phase->register_new_node(fwdraw, if_fwd);
1547     Node* fwd = new CheckCastPPNode(NULL, fwdraw, val->bottom_type());
1548     phase->register_new_node(fwd, if_fwd);
1549 
1550     // Wire up not-equal-path in slots 3.
1551     region->init_req(_fwded, if_fwd);
1552     val_phi->init_req(_fwded, fwd);
1553     raw_mem_phi->init_req(_fwded, raw_mem);
1554 
1555     // Call lrb-stub and wire up that path in slots 4
1556     Node* result_mem = NULL;
1557     ctrl = if_not_fwd;
1558     fwd = new_val;
1559     call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase);
1560     region->init_req(_evac_path, ctrl);
1561     val_phi->init_req(_evac_path, fwd);
1562     raw_mem_phi->init_req(_evac_path, result_mem);
1563 
1564     phase->register_control(region, loop, heap_stable_iff);
1565     Node* out_val = val_phi;
1566     phase->register_new_node(val_phi, region);
1567     phase->register_new_node(raw_mem_phi, region);
1568 
1569     fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
1570 
1571     ctrl = orig_ctrl;
1572 
1573     if (unc != NULL) {
1574       for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
1575         Node* u = val->fast_out(i);
1576         Node* c = phase->ctrl_or_self(u);
1577         if (u != lrb && (c != ctrl || is_dominator_same_ctrl(c, lrb, u, phase))) {
1578           phase->igvn().rehash_node_delayed(u);
1579           int nb = u->replace_edge(val, out_val);


2982                (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) ||
2983                (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) ||
2984                u->Opcode() == Op_CallLeaf, "%s", u->Name());
2985         if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
2986           if (mm == NULL) {
2987             mm = allocate_merge_mem(mem, rep_proj, rep_ctrl);
2988           }
2989           _phase->igvn().replace_input_of(u, u->find_edge(mem), mm);
2990           --i;
2991         }
2992       } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
2993         if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
2994           _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj);
2995           --i;
2996         }
2997       }
2998     }
2999   }
3000 }
3001 
3002 ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj)
3003 : Node(ctrl, obj) {
3004   ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this);
3005 }
3006 
3007 const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const {
3008   if (in(ValueIn) == NULL || in(ValueIn)->is_top()) {
3009     return Type::TOP;
3010   }
3011   const Type* t = in(ValueIn)->bottom_type();
3012   if (t == TypePtr::NULL_PTR) {
3013     return t;
3014   }
3015   return t->is_oopptr();
3016 }
3017 
3018 const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const {
3019   // Either input is TOP ==> the result is TOP
3020   const Type *t2 = phase->type(in(ValueIn));
3021   if( t2 == Type::TOP ) return Type::TOP;
3022 
3023   if (t2 == TypePtr::NULL_PTR) {




1065   phase->register_new_node(in_cset_fast_test_adr, ctrl);
1066   uint in_cset_fast_test_idx = Compile::AliasIdxRaw;
1067   const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument
1068   debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx));
1069   Node* in_cset_fast_test_load = new LoadBNode(ctrl, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BYTE, MemNode::unordered);
1070   phase->register_new_node(in_cset_fast_test_load, ctrl);
1071   Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT));
1072   phase->register_new_node(in_cset_fast_test_cmp, ctrl);
1073   Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::eq);
1074   phase->register_new_node(in_cset_fast_test_test, ctrl);
1075   IfNode* in_cset_fast_test_iff = new IfNode(ctrl, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
1076   phase->register_control(in_cset_fast_test_iff, loop, ctrl);
1077 
1078   not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff);
1079   phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff);
1080 
1081   ctrl = new IfFalseNode(in_cset_fast_test_iff);
1082   phase->register_control(ctrl, loop, in_cset_fast_test_iff);
1083 }
1084 
1085 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase) {
1086   IdealLoopTree*loop = phase->get_loop(ctrl);
1087   const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
1088 
1089   // The slow path stub consumes and produces raw memory in addition
1090   // to the existing memory edges
1091   Node* base = find_bottom_mem(ctrl, phase);
1092   MergeMemNode* mm = MergeMemNode::make(base);
1093   mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
1094   phase->register_new_node(mm, ctrl);
1095 
1096   address target = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ?
1097           CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow_JRT) :
1098           CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_JRT);
1099 
1100   Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), target, "shenandoah_load_reference_barrier", TypeRawPtr::BOTTOM);
1101   call->init_req(TypeFunc::Control, ctrl);
1102   call->init_req(TypeFunc::I_O, phase->C->top());
1103   call->init_req(TypeFunc::Memory, mm);
1104   call->init_req(TypeFunc::FramePtr, phase->C->top());
1105   call->init_req(TypeFunc::ReturnAdr, phase->C->top());
1106   call->init_req(TypeFunc::Parms, val);
1107   call->init_req(TypeFunc::Parms+1, load_addr);
1108   phase->register_control(call, loop, ctrl);
1109   ctrl = new ProjNode(call, TypeFunc::Control);
1110   phase->register_control(ctrl, loop, call);
1111   result_mem = new ProjNode(call, TypeFunc::Memory);
1112   phase->register_new_node(result_mem, call);
1113   val = new ProjNode(call, TypeFunc::Parms);
1114   phase->register_new_node(val, call);
1115   val = new CheckCastPPNode(ctrl, val, obj_type);
1116   phase->register_new_node(val, ctrl);
1117 }
1118 
1119 void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
1120   Node* ctrl = phase->get_ctrl(barrier);
1121   Node* init_raw_mem = fixer.find_mem(ctrl, barrier);
1122 
1123   // Update the control of all nodes that should be after the
1124   // barrier control flow
1125   uses.clear();
1126   // Every node that is control dependent on the barrier's input
1127   // control will be after the expanded barrier. The raw memory (if


1452     Node* unc_ctrl = NULL;
1453     if (unc != NULL) {
1454       if (val->in(ShenandoahLoadReferenceBarrierNode::Control) != ctrl) {
1455         unc = NULL;
1456       } else {
1457         unc_ctrl = val->in(ShenandoahLoadReferenceBarrierNode::Control);
1458       }
1459     }
1460 
1461     Node* uncasted_val = val;
1462     if (unc != NULL) {
1463       uncasted_val = val->in(1);
1464     }
1465 
1466     Node* heap_stable_ctrl = NULL;
1467     Node* null_ctrl = NULL;
1468 
1469     assert(val->bottom_type()->make_oopptr(), "need oop");
1470     assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
1471 
1472     enum { _heap_stable = 1, _not_cset, _evac_path, _null_path, PATH_LIMIT };
1473     Node* region = new RegionNode(PATH_LIMIT);
1474     Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
1475     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1476 
1477     // Stable path.
1478     test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
1479     IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
1480 
1481     // Heap stable case
1482     region->init_req(_heap_stable, heap_stable_ctrl);
1483     val_phi->init_req(_heap_stable, uncasted_val);
1484     raw_mem_phi->init_req(_heap_stable, raw_mem);
1485 
1486     Node* reg2_ctrl = NULL;
1487     // Null case
1488     test_null(ctrl, val, null_ctrl, phase);
1489     if (null_ctrl != NULL) {
1490       reg2_ctrl = null_ctrl->in(0);
1491       region->init_req(_null_path, null_ctrl);
1492       val_phi->init_req(_null_path, uncasted_val);


1502     Node* not_cset_ctrl = NULL;
1503     in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase);
1504     if (not_cset_ctrl != NULL) {
1505       if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0);
1506       region->init_req(_not_cset, not_cset_ctrl);
1507       val_phi->init_req(_not_cset, uncasted_val);
1508       raw_mem_phi->init_req(_not_cset, raw_mem);
1509     }
1510 
1511     // Resolve object when orig-value is in cset.
1512     // Make the unconditional resolve for fwdptr.
1513     Node* new_val = uncasted_val;
1514     if (unc_ctrl != NULL) {
1515       // Clone the null check in this branch to allow implicit null check
1516       new_val = clone_null_check(ctrl, val, unc_ctrl, phase);
1517       fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
1518 
1519       IfNode* iff = unc_ctrl->in(0)->as_If();
1520       phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
1521     }





































1522 
1523     // Call lrb-stub and wire up that path in slots 4
1524     Node* result_mem = NULL;
1525     Node* fwd = new_val;
1526     call_lrb_stub(ctrl, fwd, lrb->in(ShenandoahLoadReferenceBarrierNode::LoadAddr), result_mem, raw_mem, phase);

1527     region->init_req(_evac_path, ctrl);
1528     val_phi->init_req(_evac_path, fwd);
1529     raw_mem_phi->init_req(_evac_path, result_mem);
1530 
1531     phase->register_control(region, loop, heap_stable_iff);
1532     Node* out_val = val_phi;
1533     phase->register_new_node(val_phi, region);
1534     phase->register_new_node(raw_mem_phi, region);
1535 
1536     fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
1537 
1538     ctrl = orig_ctrl;
1539 
1540     if (unc != NULL) {
1541       for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
1542         Node* u = val->fast_out(i);
1543         Node* c = phase->ctrl_or_self(u);
1544         if (u != lrb && (c != ctrl || is_dominator_same_ctrl(c, lrb, u, phase))) {
1545           phase->igvn().rehash_node_delayed(u);
1546           int nb = u->replace_edge(val, out_val);


2949                (u->is_CallStaticJava() && u->as_CallStaticJava()->uncommon_trap_request() != 0) ||
2950                (u->is_CallStaticJava() && u->as_CallStaticJava()->_entry_point == OptoRuntime::rethrow_stub()) ||
2951                u->Opcode() == Op_CallLeaf, "%s", u->Name());
2952         if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
2953           if (mm == NULL) {
2954             mm = allocate_merge_mem(mem, rep_proj, rep_ctrl);
2955           }
2956           _phase->igvn().replace_input_of(u, u->find_edge(mem), mm);
2957           --i;
2958         }
2959       } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
2960         if (ShenandoahBarrierC2Support::is_dominator(rep_ctrl, _phase->ctrl_or_self(u), replacement, u, _phase)) {
2961           _phase->igvn().replace_input_of(u, u->find_edge(mem), rep_proj);
2962           --i;
2963         }
2964       }
2965     }
2966   }
2967 }
2968 
2969 ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, Node* load_addr)
2970 : Node(ctrl, obj, load_addr) {
2971   ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this);
2972 }
2973 
2974 const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const {
2975   if (in(ValueIn) == NULL || in(ValueIn)->is_top()) {
2976     return Type::TOP;
2977   }
2978   const Type* t = in(ValueIn)->bottom_type();
2979   if (t == TypePtr::NULL_PTR) {
2980     return t;
2981   }
2982   return t->is_oopptr();
2983 }
2984 
2985 const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const {
2986   // Either input is TOP ==> the result is TOP
2987   const Type *t2 = phase->type(in(ValueIn));
2988   if( t2 == Type::TOP ) return Type::TOP;
2989 
2990   if (t2 == TypePtr::NULL_PTR) {


< prev index next >