< prev index next >

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

Print this page




   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 
  26 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
  27 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
  28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  29 #include "gc/shenandoah/shenandoahForwarding.hpp"
  30 #include "gc/shenandoah/shenandoahHeap.hpp"
  31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  32 #include "gc/shenandoah/shenandoahRuntime.hpp"
  33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  34 #include "opto/arraycopynode.hpp"
  35 #include "opto/block.hpp"
  36 #include "opto/callnode.hpp"
  37 #include "opto/castnode.hpp"
  38 #include "opto/movenode.hpp"
  39 #include "opto/phaseX.hpp"
  40 #include "opto/rootnode.hpp"
  41 #include "opto/runtime.hpp"
  42 #include "opto/subnode.hpp"
  43 
  44 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
  45   ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
  46   if ((state->enqueue_barriers_count() +
  47        state->load_reference_barriers_count()) > 0) {
  48     bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;
  49     C->clear_major_progress();


1316             --i; imax -= nb;
1317           }
1318 
1319           for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
1320             Node* u = val->fast_out(i);
1321             if (u == lrb) {
1322               continue;
1323             }
1324             phase->igvn().rehash_node_delayed(u);
1325             int nb = u->replace_edge(val, new_cast);
1326             assert(nb > 0, "no update?");
1327             --i; imax -= nb;
1328           }
1329 
1330           ctrl = unc_ctrl_clone;
1331           phase->set_ctrl_and_loop(lrb, ctrl);
1332           break;
1333         }
1334       }
1335     }
1336     if ((ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) || ctrl->is_CallJava()) {
1337       CallNode* call = ctrl->is_Proj() ? ctrl->in(0)->as_CallJava() : ctrl->as_CallJava();
1338       CallProjections projs;
1339       call->extract_projections(&projs, false, false);
1340 
1341       Node* lrb_clone = lrb->clone();
1342       phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1343       phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1344 
1345       stack.push(lrb, 0);
1346       clones.push(lrb_clone);
1347 
1348       do {
1349         assert(stack.size() == clones.size(), "");
1350         Node* n = stack.node();
1351 #ifdef ASSERT
1352         if (n->is_Load()) {
1353           Node* mem = n->in(MemNode::Memory);
1354           for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1355             Node* u = mem->fast_out(j);
1356             assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1357           }
1358         }
1359 #endif
1360         uint idx = stack.index();
1361         Node* n_clone = clones.at(clones.size()-1);
1362         if (idx < n->outcnt()) {
1363           Node* u = n->raw_out(idx);
1364           Node* c = phase->ctrl_or_self(u);
1365           if (phase->is_dominator(call, c) && phase->is_dominator(c, projs.fallthrough_proj)) {
1366             stack.set_index(idx+1);
1367             assert(!u->is_CFG(), "");
1368             stack.push(u, 0);
1369             Node* u_clone = u->clone();
1370             int nb = u_clone->replace_edge(n, n_clone);
1371             assert(nb > 0, "should have replaced some uses");
1372             phase->register_new_node(u_clone, projs.catchall_catchproj);
1373             clones.push(u_clone);
1374             phase->set_ctrl(u, projs.fallthrough_catchproj);
1375           } else {
1376             bool replaced = false;
1377             if (u->is_Phi()) {
1378               for (uint k = 1; k < u->req(); k++) {
1379                 if (u->in(k) == n) {
1380                   if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1381                     phase->igvn().replace_input_of(u, k, n_clone);
1382                     replaced = true;
1383                   } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1384                     phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1385                     replaced = true;


1387                 }
1388               }
1389             } else {
1390               if (phase->is_dominator(projs.catchall_catchproj, c)) {
1391                 phase->igvn().rehash_node_delayed(u);
1392                 int nb = u->replace_edge(n, n_clone);
1393                 assert(nb > 0, "should have replaced some uses");
1394                 replaced = true;
1395               } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1396                 phase->igvn().rehash_node_delayed(u);
1397                 int nb = u->replace_edge(n, create_phis_on_call_return(ctrl, c, n, n_clone, projs, phase));
1398                 assert(nb > 0, "should have replaced some uses");
1399                 replaced = true;
1400               }
1401             }
1402             if (!replaced) {
1403               stack.set_index(idx+1);
1404             }
1405           }
1406         } else {


1407           stack.pop();
1408           clones.pop();
1409         }
1410       } while (stack.size() > 0);
1411       assert(stack.size() == 0 && clones.size() == 0, "");

1412     }
1413   }
1414 
1415   // Expand load-reference-barriers
1416   MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase);
1417   Unique_Node_List uses_to_ignore;
1418   for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1419     ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1420     if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) {
1421       phase->igvn().replace_node(lrb, lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
1422       continue;
1423     }
1424     uint last = phase->C->unique();
1425     Node* ctrl = phase->get_ctrl(lrb);
1426     Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1427 
1428 
1429     Node* orig_ctrl = ctrl;
1430 
1431     Node* raw_mem = fixer.find_mem(ctrl, lrb);


1488     Node* not_cset_ctrl = NULL;
1489     in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase);
1490     if (not_cset_ctrl != NULL) {
1491       if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0);
1492       region->init_req(_not_cset, not_cset_ctrl);
1493       val_phi->init_req(_not_cset, uncasted_val);
1494       raw_mem_phi->init_req(_not_cset, raw_mem);
1495     }
1496 
1497     // Resolve object when orig-value is in cset.
1498     // Make the unconditional resolve for fwdptr.
1499     Node* new_val = uncasted_val;
1500     if (unc_ctrl != NULL) {
1501       // Clone the null check in this branch to allow implicit null check
1502       new_val = clone_null_check(ctrl, val, unc_ctrl, phase);
1503       fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
1504 
1505       IfNode* iff = unc_ctrl->in(0)->as_If();
1506       phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
1507     }
1508     Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahForwarding::byte_offset()));
1509     phase->register_new_node(addr, ctrl);
1510     assert(val->bottom_type()->isa_oopptr(), "what else?");
1511     const TypePtr* obj_type =  val->bottom_type()->is_oopptr();
1512     const TypePtr* adr_type = TypeRawPtr::BOTTOM;
1513     Node* fwd = new LoadPNode(ctrl, raw_mem, addr, adr_type, obj_type, MemNode::unordered);
1514     phase->register_new_node(fwd, ctrl);
1515 
1516     // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr
1517     Node* cmp = new CmpPNode(fwd, new_val);
1518     phase->register_new_node(cmp, ctrl);
1519     Node* bol = new BoolNode(cmp, BoolTest::eq);
1520     phase->register_new_node(bol, ctrl);
1521 
1522     IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
1523     if (reg2_ctrl == NULL) reg2_ctrl = iff;
1524     phase->register_control(iff, loop, ctrl);
1525     Node* if_not_eq = new IfFalseNode(iff);
1526     phase->register_control(if_not_eq, loop, iff);
1527     Node* if_eq = new IfTrueNode(iff);
1528     phase->register_control(if_eq, loop, iff);


1930   }
1931 
1932   if (!phase->C->major_progress()) {
1933     VectorSet seen(Thread::current()->resource_area());
1934     for (uint i = 0; i < heap_stable_tests.size(); i++) {
1935       Node* n = heap_stable_tests.at(i);
1936       IdealLoopTree* loop = phase->get_loop(n);
1937       if (loop != phase->ltree_root() &&
1938           loop->_child == NULL &&
1939           !loop->_irreducible) {
1940         LoopNode* head = loop->_head->as_Loop();
1941         if ((!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) &&
1942             !seen.test_set(head->_idx)) {
1943           IfNode* iff = find_unswitching_candidate(loop, phase);
1944           if (iff != NULL) {
1945             Node* bol = iff->in(1);
1946             if (head->is_strip_mined()) {
1947               head->verify_strip_mined(0);
1948             }
1949             move_heap_stable_test_out_of_loop(iff, phase);
1950 
1951             AutoNodeBudget node_budget(phase);
1952 
1953             if (loop->policy_unswitching(phase)) {
1954               if (head->is_strip_mined()) {
1955                 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop();
1956                 hide_strip_mined_loop(outer, head->as_CountedLoop(), phase);
1957               }
1958               phase->do_unswitching(loop, old_new);
1959             } else {
1960               // Not proceeding with unswitching. Move load back in
1961               // the loop.
1962               phase->igvn().replace_input_of(iff, 1, bol);
1963             }
1964           }
1965         }
1966       }
1967     }
1968   }
1969 }
1970 
1971 #ifdef ASSERT
1972 void ShenandoahBarrierC2Support::verify_raw_mem(RootNode* root) {


3106     if (visited.member(n)) {
3107       stack.pop();
3108       continue;
3109     }
3110     visited.push(n);
3111     bool visit_users = false;
3112     switch (n->Opcode()) {
3113       case Op_StoreN:
3114       case Op_StoreP: {
3115         strength = STRONG;
3116         break;
3117       }
3118       case Op_CmpP: {
3119         if (!n->in(1)->bottom_type()->higher_equal(TypePtr::NULL_PTR) &&
3120             !n->in(2)->bottom_type()->higher_equal(TypePtr::NULL_PTR)) {
3121           strength = STRONG;
3122         }
3123         break;
3124       }
3125       case Op_CallStaticJava: {
3126         // If it's an deopt-call we don't need barriers because
3127         // the LRB will be applied when unpacking the deopt frame.
3128         if (n->as_CallStaticJava()->uncommon_trap_request() == 0) {
3129           strength = STRONG;
3130         }
3131         break;
3132       }
3133       case Op_CallDynamicJava:
3134       case Op_CallLeaf:
3135       case Op_CallLeafNoFP:
3136       case Op_CompareAndSwapL:
3137       case Op_CompareAndSwapI:
3138       case Op_CompareAndSwapB:
3139       case Op_CompareAndSwapS:
3140       case Op_CompareAndSwapN:
3141       case Op_CompareAndSwapP:




3142       case Op_CompareAndExchangeL:
3143       case Op_CompareAndExchangeI:
3144       case Op_CompareAndExchangeB:
3145       case Op_CompareAndExchangeS:
3146       case Op_CompareAndExchangeN:
3147       case Op_CompareAndExchangeP:
3148       case Op_WeakCompareAndSwapL:
3149       case Op_WeakCompareAndSwapI:
3150       case Op_WeakCompareAndSwapB:
3151       case Op_WeakCompareAndSwapS:
3152       case Op_WeakCompareAndSwapN:
3153       case Op_WeakCompareAndSwapP:
3154       case Op_ShenandoahCompareAndSwapN:
3155       case Op_ShenandoahCompareAndSwapP:
3156       case Op_ShenandoahWeakCompareAndSwapN:
3157       case Op_ShenandoahWeakCompareAndSwapP:
3158       case Op_ShenandoahCompareAndExchangeN:
3159       case Op_ShenandoahCompareAndExchangeP:
3160       case Op_GetAndSetL:
3161       case Op_GetAndSetI:
3162       case Op_GetAndSetB:
3163       case Op_GetAndSetS:
3164       case Op_GetAndSetP:
3165       case Op_GetAndSetN:
3166       case Op_GetAndAddL:
3167       case Op_GetAndAddI:
3168       case Op_GetAndAddB:
3169       case Op_GetAndAddS:
3170       case Op_ShenandoahEnqueueBarrier:
3171       case Op_FastLock:
3172       case Op_FastUnlock:
3173       case Op_Rethrow:
3174       case Op_Return:
3175       case Op_StoreB:
3176       case Op_StoreC:
3177       case Op_StoreD:
3178       case Op_StoreF:
3179       case Op_StoreL:




   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 
  26 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
  27 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
  28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  29 #include "gc/shenandoah/shenandoahBrooksPointer.hpp"
  30 #include "gc/shenandoah/shenandoahHeap.hpp"
  31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  32 #include "gc/shenandoah/shenandoahRuntime.hpp"
  33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  34 #include "opto/arraycopynode.hpp"
  35 #include "opto/block.hpp"
  36 #include "opto/callnode.hpp"
  37 #include "opto/castnode.hpp"
  38 #include "opto/movenode.hpp"
  39 #include "opto/phaseX.hpp"
  40 #include "opto/rootnode.hpp"
  41 #include "opto/runtime.hpp"
  42 #include "opto/subnode.hpp"
  43 
  44 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
  45   ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
  46   if ((state->enqueue_barriers_count() +
  47        state->load_reference_barriers_count()) > 0) {
  48     bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;
  49     C->clear_major_progress();


1316             --i; imax -= nb;
1317           }
1318 
1319           for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
1320             Node* u = val->fast_out(i);
1321             if (u == lrb) {
1322               continue;
1323             }
1324             phase->igvn().rehash_node_delayed(u);
1325             int nb = u->replace_edge(val, new_cast);
1326             assert(nb > 0, "no update?");
1327             --i; imax -= nb;
1328           }
1329 
1330           ctrl = unc_ctrl_clone;
1331           phase->set_ctrl_and_loop(lrb, ctrl);
1332           break;
1333         }
1334       }
1335     }
1336     if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
1337       CallNode* call = ctrl->in(0)->as_CallJava();
1338       CallProjections projs;
1339       call->extract_projections(&projs, false, false);
1340 
1341       Node* lrb_clone = lrb->clone();
1342       phase->register_new_node(lrb_clone, projs.catchall_catchproj);
1343       phase->set_ctrl(lrb, projs.fallthrough_catchproj);
1344 
1345       stack.push(lrb, 0);
1346       clones.push(lrb_clone);
1347 
1348       do {
1349         assert(stack.size() == clones.size(), "");
1350         Node* n = stack.node();
1351 #ifdef ASSERT
1352         if (n->is_Load()) {
1353           Node* mem = n->in(MemNode::Memory);
1354           for (DUIterator_Fast jmax, j = mem->fast_outs(jmax); j < jmax; j++) {
1355             Node* u = mem->fast_out(j);
1356             assert(!u->is_Store() || !u->is_LoadStore() || phase->get_ctrl(u) != ctrl, "anti dependent store?");
1357           }
1358         }
1359 #endif
1360         uint idx = stack.index();
1361         Node* n_clone = clones.at(clones.size()-1);
1362         if (idx < n->outcnt()) {
1363           Node* u = n->raw_out(idx);
1364           Node* c = phase->ctrl_or_self(u);
1365           if (c == ctrl) {
1366             stack.set_index(idx+1);
1367             assert(!u->is_CFG(), "");
1368             stack.push(u, 0);
1369             Node* u_clone = u->clone();
1370             int nb = u_clone->replace_edge(n, n_clone);
1371             assert(nb > 0, "should have replaced some uses");
1372             phase->register_new_node(u_clone, projs.catchall_catchproj);
1373             clones.push(u_clone);
1374             phase->set_ctrl(u, projs.fallthrough_catchproj);
1375           } else {
1376             bool replaced = false;
1377             if (u->is_Phi()) {
1378               for (uint k = 1; k < u->req(); k++) {
1379                 if (u->in(k) == n) {
1380                   if (phase->is_dominator(projs.catchall_catchproj, u->in(0)->in(k))) {
1381                     phase->igvn().replace_input_of(u, k, n_clone);
1382                     replaced = true;
1383                   } else if (!phase->is_dominator(projs.fallthrough_catchproj, u->in(0)->in(k))) {
1384                     phase->igvn().replace_input_of(u, k, create_phis_on_call_return(ctrl, u->in(0)->in(k), n, n_clone, projs, phase));
1385                     replaced = true;


1387                 }
1388               }
1389             } else {
1390               if (phase->is_dominator(projs.catchall_catchproj, c)) {
1391                 phase->igvn().rehash_node_delayed(u);
1392                 int nb = u->replace_edge(n, n_clone);
1393                 assert(nb > 0, "should have replaced some uses");
1394                 replaced = true;
1395               } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1396                 phase->igvn().rehash_node_delayed(u);
1397                 int nb = u->replace_edge(n, create_phis_on_call_return(ctrl, c, n, n_clone, projs, phase));
1398                 assert(nb > 0, "should have replaced some uses");
1399                 replaced = true;
1400               }
1401             }
1402             if (!replaced) {
1403               stack.set_index(idx+1);
1404             }
1405           }
1406         } else {
1407           // assert(n_clone->outcnt() > 0, "");
1408           // assert(n->outcnt() > 0, "");
1409           stack.pop();
1410           clones.pop();
1411         }
1412       } while (stack.size() > 0);
1413       assert(stack.size() == 0 && clones.size() == 0, "");
1414       ctrl = projs.fallthrough_catchproj;
1415     }
1416   }
1417 
1418   // Expand load-reference-barriers
1419   MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase);
1420   Unique_Node_List uses_to_ignore;
1421   for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1422     ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1423     if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) {
1424       phase->igvn().replace_node(lrb, lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
1425       continue;
1426     }
1427     uint last = phase->C->unique();
1428     Node* ctrl = phase->get_ctrl(lrb);
1429     Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1430 
1431 
1432     Node* orig_ctrl = ctrl;
1433 
1434     Node* raw_mem = fixer.find_mem(ctrl, lrb);


1491     Node* not_cset_ctrl = NULL;
1492     in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase);
1493     if (not_cset_ctrl != NULL) {
1494       if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0);
1495       region->init_req(_not_cset, not_cset_ctrl);
1496       val_phi->init_req(_not_cset, uncasted_val);
1497       raw_mem_phi->init_req(_not_cset, raw_mem);
1498     }
1499 
1500     // Resolve object when orig-value is in cset.
1501     // Make the unconditional resolve for fwdptr.
1502     Node* new_val = uncasted_val;
1503     if (unc_ctrl != NULL) {
1504       // Clone the null check in this branch to allow implicit null check
1505       new_val = clone_null_check(ctrl, val, unc_ctrl, phase);
1506       fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase);
1507 
1508       IfNode* iff = unc_ctrl->in(0)->as_If();
1509       phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
1510     }
1511     Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahBrooksPointer::byte_offset()));
1512     phase->register_new_node(addr, ctrl);
1513     assert(val->bottom_type()->isa_oopptr(), "what else?");
1514     const TypePtr* obj_type =  val->bottom_type()->is_oopptr();
1515     const TypePtr* adr_type = TypeRawPtr::BOTTOM;
1516     Node* fwd = new LoadPNode(ctrl, raw_mem, addr, adr_type, obj_type, MemNode::unordered);
1517     phase->register_new_node(fwd, ctrl);
1518 
1519     // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr
1520     Node* cmp = new CmpPNode(fwd, new_val);
1521     phase->register_new_node(cmp, ctrl);
1522     Node* bol = new BoolNode(cmp, BoolTest::eq);
1523     phase->register_new_node(bol, ctrl);
1524 
1525     IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
1526     if (reg2_ctrl == NULL) reg2_ctrl = iff;
1527     phase->register_control(iff, loop, ctrl);
1528     Node* if_not_eq = new IfFalseNode(iff);
1529     phase->register_control(if_not_eq, loop, iff);
1530     Node* if_eq = new IfTrueNode(iff);
1531     phase->register_control(if_eq, loop, iff);


1933   }
1934 
1935   if (!phase->C->major_progress()) {
1936     VectorSet seen(Thread::current()->resource_area());
1937     for (uint i = 0; i < heap_stable_tests.size(); i++) {
1938       Node* n = heap_stable_tests.at(i);
1939       IdealLoopTree* loop = phase->get_loop(n);
1940       if (loop != phase->ltree_root() &&
1941           loop->_child == NULL &&
1942           !loop->_irreducible) {
1943         LoopNode* head = loop->_head->as_Loop();
1944         if ((!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) &&
1945             !seen.test_set(head->_idx)) {
1946           IfNode* iff = find_unswitching_candidate(loop, phase);
1947           if (iff != NULL) {
1948             Node* bol = iff->in(1);
1949             if (head->is_strip_mined()) {
1950               head->verify_strip_mined(0);
1951             }
1952             move_heap_stable_test_out_of_loop(iff, phase);



1953             if (loop->policy_unswitching(phase)) {
1954               if (head->is_strip_mined()) {
1955                 OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop();
1956                 hide_strip_mined_loop(outer, head->as_CountedLoop(), phase);
1957               }
1958               phase->do_unswitching(loop, old_new);
1959             } else {
1960               // Not proceeding with unswitching. Move load back in
1961               // the loop.
1962               phase->igvn().replace_input_of(iff, 1, bol);
1963             }
1964           }
1965         }
1966       }
1967     }
1968   }
1969 }
1970 
1971 #ifdef ASSERT
1972 void ShenandoahBarrierC2Support::verify_raw_mem(RootNode* root) {


3106     if (visited.member(n)) {
3107       stack.pop();
3108       continue;
3109     }
3110     visited.push(n);
3111     bool visit_users = false;
3112     switch (n->Opcode()) {
3113       case Op_StoreN:
3114       case Op_StoreP: {
3115         strength = STRONG;
3116         break;
3117       }
3118       case Op_CmpP: {
3119         if (!n->in(1)->bottom_type()->higher_equal(TypePtr::NULL_PTR) &&
3120             !n->in(2)->bottom_type()->higher_equal(TypePtr::NULL_PTR)) {
3121           strength = STRONG;
3122         }
3123         break;
3124       }
3125       case Op_CallStaticJava: {
3126         strength = STRONG;




3127         break;
3128       }
3129       case Op_CallDynamicJava:
3130       case Op_CallLeaf:
3131       case Op_CallLeafNoFP:
3132       case Op_CompareAndSwapL:
3133       case Op_CompareAndSwapI:
3134       case Op_CompareAndSwapB:
3135       case Op_CompareAndSwapS:
3136       case Op_ShenandoahCompareAndSwapN:
3137       case Op_ShenandoahCompareAndSwapP:
3138       case Op_ShenandoahWeakCompareAndSwapN:
3139       case Op_ShenandoahWeakCompareAndSwapP:
3140       case Op_ShenandoahCompareAndExchangeN:
3141       case Op_ShenandoahCompareAndExchangeP:
3142       case Op_CompareAndExchangeL:
3143       case Op_CompareAndExchangeI:
3144       case Op_CompareAndExchangeB:
3145       case Op_CompareAndExchangeS:


3146       case Op_WeakCompareAndSwapL:
3147       case Op_WeakCompareAndSwapI:
3148       case Op_WeakCompareAndSwapB:
3149       case Op_WeakCompareAndSwapS:








3150       case Op_GetAndSetL:
3151       case Op_GetAndSetI:
3152       case Op_GetAndSetB:
3153       case Op_GetAndSetS:
3154       case Op_GetAndSetP:
3155       case Op_GetAndSetN:
3156       case Op_GetAndAddL:
3157       case Op_GetAndAddI:
3158       case Op_GetAndAddB:
3159       case Op_GetAndAddS:
3160       case Op_ShenandoahEnqueueBarrier:
3161       case Op_FastLock:
3162       case Op_FastUnlock:
3163       case Op_Rethrow:
3164       case Op_Return:
3165       case Op_StoreB:
3166       case Op_StoreC:
3167       case Op_StoreD:
3168       case Op_StoreF:
3169       case Op_StoreL:


< prev index next >