< prev index next >

src/hotspot/share/opto/parse2.cpp

Print this page

   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 

  25 #include "ci/ciMethodData.hpp"

  26 #include "classfile/vmSymbols.hpp"
  27 #include "compiler/compileLog.hpp"
  28 #include "interpreter/linkResolver.hpp"
  29 #include "jvm_io.h"
  30 #include "memory/resourceArea.hpp"
  31 #include "memory/universe.hpp"
  32 #include "oops/oop.inline.hpp"
  33 #include "opto/addnode.hpp"
  34 #include "opto/castnode.hpp"
  35 #include "opto/convertnode.hpp"
  36 #include "opto/divnode.hpp"
  37 #include "opto/idealGraphPrinter.hpp"


  38 #include "opto/matcher.hpp"
  39 #include "opto/memnode.hpp"
  40 #include "opto/mulnode.hpp"
  41 #include "opto/opaquenode.hpp"
  42 #include "opto/parse.hpp"
  43 #include "opto/runtime.hpp"
  44 #include "runtime/deoptimization.hpp"
  45 #include "runtime/sharedRuntime.hpp"
  46 
  47 #ifndef PRODUCT
  48 extern uint explicit_null_checks_inserted,
  49             explicit_null_checks_elided;
  50 #endif
  51 

















  52 //---------------------------------array_load----------------------------------
  53 void Parse::array_load(BasicType bt) {
  54   const Type* elemtype = Type::TOP;
  55   bool big_val = bt == T_DOUBLE || bt == T_LONG;
  56   Node* adr = array_addressing(bt, 0, elemtype);
  57   if (stopped())  return;     // guaranteed null or range check
  58 
  59   pop();                      // index (already used)
  60   Node* array = pop();        // the array itself




























































  61 
  62   if (elemtype == TypeInt::BOOL) {
  63     bt = T_BOOLEAN;
  64   }
  65   const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt);
  66 
  67   Node* ld = access_load_at(array, adr, adr_type, elemtype, bt,
  68                             IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD);
  69   if (big_val) {
  70     push_pair(ld);
  71   } else {
  72     push(ld);

  73   }

  74 }
  75 




























  76 
  77 //--------------------------------array_store----------------------------------
  78 void Parse::array_store(BasicType bt) {
  79   const Type* elemtype = Type::TOP;
  80   bool big_val = bt == T_DOUBLE || bt == T_LONG;
  81   Node* adr = array_addressing(bt, big_val ? 2 : 1, elemtype);
  82   if (stopped())  return;     // guaranteed null or range check

  83   if (bt == T_OBJECT) {
  84     array_store_check();
  85     if (stopped()) {
  86       return;
  87     }
  88   }
  89   Node* val;                  // Oop to store
  90   if (big_val) {
  91     val = pop_pair();
  92   } else {
  93     val = pop();
  94   }
  95   pop();                      // index (already used)
  96   Node* array = pop();        // the array itself
  97 
  98   if (elemtype == TypeInt::BOOL) {
  99     bt = T_BOOLEAN;
 100   }
 101   const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt);








































































 102 
 103   access_store_at(array, adr, adr_type, val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY);



















 104 }
 105 

























 106 
 107 //------------------------------array_addressing-------------------------------
 108 // Pull array and index from the stack.  Compute pointer-to-element.
 109 Node* Parse::array_addressing(BasicType type, int vals, const Type*& elemtype) {
 110   Node *idx   = peek(0+vals);   // Get from stack without popping
 111   Node *ary   = peek(1+vals);   // in case of exception
 112 
 113   // Null check the array base, with correct stack contents
 114   ary = null_check(ary, T_ARRAY);
 115   // Compile-time detect of null-exception?
 116   if (stopped())  return top();
 117 
 118   const TypeAryPtr* arytype  = _gvn.type(ary)->is_aryptr();
 119   const TypeInt*    sizetype = arytype->size();
 120   elemtype = arytype->elem();
 121 
 122   if (UseUniqueSubclasses) {
 123     const Type* el = elemtype->make_ptr();
 124     if (el && el->isa_instptr()) {
 125       const TypeInstPtr* toop = el->is_instptr();
 126       if (toop->instance_klass()->unique_concrete_subklass()) {
 127         // If we load from "AbstractClass[]" we must see "ConcreteSubClass".
 128         const Type* subklass = Type::get_const_type(toop->instance_klass());
 129         elemtype = subklass->join_speculative(el);
 130       }
 131     }
 132   }
 133 
 134   // Check for big class initializers with all constant offsets
 135   // feeding into a known-size array.
 136   const TypeInt* idxtype = _gvn.type(idx)->is_int();
 137   // See if the highest idx value is less than the lowest array bound,
 138   // and if the idx value cannot be negative:
 139   bool need_range_check = true;
 140   if (idxtype->_hi < sizetype->_lo && idxtype->_lo >= 0) {
 141     need_range_check = false;
 142     if (C->log() != nullptr)   C->log()->elem("observe that='!need_range_check'");
 143   }
 144 
 145   if (!arytype->is_loaded()) {
 146     // Only fails for some -Xcomp runs
 147     // The class is unloaded.  We have to run this bytecode in the interpreter.
 148     ciKlass* klass = arytype->unloaded_klass();
 149 
 150     uncommon_trap(Deoptimization::Reason_unloaded,
 151                   Deoptimization::Action_reinterpret,
 152                   klass, "!loaded array");
 153     return top();
 154   }
 155 
 156   // Do the range check
 157   if (need_range_check) {
 158     Node* tst;
 159     if (sizetype->_hi <= 0) {
 160       // The greatest array bound is negative, so we can conclude that we're
 161       // compiling unreachable code, but the unsigned compare trick used below
 162       // only works with non-negative lengths.  Instead, hack "tst" to be zero so
 163       // the uncommon_trap path will always be taken.
 164       tst = _gvn.intcon(0);
 165     } else {
 166       // Range is constant in array-oop, so we can use the original state of mem
 167       Node* len = load_array_length(ary);
 168 
 169       // Test length vs index (standard trick using unsigned compare)
 170       Node* chk = _gvn.transform( new CmpUNode(idx, len) );
 171       BoolTest::mask btest = BoolTest::lt;
 172       tst = _gvn.transform( new BoolNode(chk, btest) );
 173     }
 174     RangeCheckNode* rc = new RangeCheckNode(control(), tst, PROB_MAX, COUNT_UNKNOWN);
 175     _gvn.set_type(rc, rc->Value(&_gvn));
 176     if (!tst->is_Con()) {
 177       record_for_igvn(rc);
 178     }
 179     set_control(_gvn.transform(new IfTrueNode(rc)));
 180     // Branch to failure if out of bounds
 181     {
 182       PreserveJVMState pjvms(this);
 183       set_control(_gvn.transform(new IfFalseNode(rc)));
 184       if (C->allow_range_check_smearing()) {
 185         // Do not use builtin_throw, since range checks are sometimes
 186         // made more stringent by an optimistic transformation.
 187         // This creates "tentative" range checks at this point,
 188         // which are not guaranteed to throw exceptions.
 189         // See IfNode::Ideal, is_range_check, adjust_check.
 190         uncommon_trap(Deoptimization::Reason_range_check,
 191                       Deoptimization::Action_make_not_entrant,
 192                       nullptr, "range_check");
 193       } else {
 194         // If we have already recompiled with the range-check-widening
 195         // heroic optimization turned off, then we must really be throwing
 196         // range check exceptions.
 197         builtin_throw(Deoptimization::Reason_range_check);
 198       }
 199     }
 200   }

 201   // Check for always knowing you are throwing a range-check exception
 202   if (stopped())  return top();
 203 
 204   // Make array address computation control dependent to prevent it
 205   // from floating above the range check during loop optimizations.
 206   Node* ptr = array_element_address(ary, idx, type, sizetype, control());
 207   assert(ptr != top(), "top should go hand-in-hand with stopped");
 208 
 209   return ptr;
 210 }
 211 







































































































































































































 212 
 213 // returns IfNode
 214 IfNode* Parse::jump_if_fork_int(Node* a, Node* b, BoolTest::mask mask, float prob, float cnt) {
 215   Node   *cmp = _gvn.transform(new CmpINode(a, b)); // two cases: shiftcount > 32 and shiftcount <= 32
 216   Node   *tst = _gvn.transform(new BoolNode(cmp, mask));
 217   IfNode *iff = create_and_map_if(control(), tst, prob, cnt);
 218   return iff;
 219 }
 220 
 221 
 222 // sentinel value for the target bci to mark never taken branches
 223 // (according to profiling)
 224 static const int never_reached = INT_MAX;
 225 
 226 //------------------------------helper for tableswitch-------------------------
 227 void Parse::jump_if_true_fork(IfNode *iff, int dest_bci_if_true, bool unc) {
 228   // True branch, use existing map info
 229   { PreserveJVMState pjvms(this);
 230     Node *iftrue  = _gvn.transform( new IfTrueNode (iff) );
 231     set_control( iftrue );

1428   // False branch
1429   Node* iffalse = _gvn.transform( new IfFalseNode(iff) );
1430   set_control(iffalse);
1431 
1432   if (stopped()) {              // Path is dead?
1433     NOT_PRODUCT(explicit_null_checks_elided++);
1434     if (C->eliminate_boxing()) {
1435       // Mark the successor block as parsed
1436       next_block->next_path_num();
1437     }
1438   } else  {                     // Path is live.
1439     adjust_map_after_if(BoolTest(btest).negate(), c, 1.0-prob, next_block);
1440   }
1441 
1442   if (do_stress_trap) {
1443     stress_trap(iff, counter, incr_store);
1444   }
1445 }
1446 
1447 //------------------------------------do_if------------------------------------
1448 void Parse::do_if(BoolTest::mask btest, Node* c) {
1449   int target_bci = iter().get_dest();
1450 
1451   Block* branch_block = successor_for_bci(target_bci);
1452   Block* next_block   = successor_for_bci(iter().next_bci());
1453 
1454   float cnt;
1455   float prob = branch_prediction(cnt, btest, target_bci, c);
1456   float untaken_prob = 1.0 - prob;
1457 
1458   if (prob == PROB_UNKNOWN) {
1459     if (PrintOpto && Verbose) {
1460       tty->print_cr("Never-taken edge stops compilation at bci %d", bci());
1461     }
1462     repush_if_args(); // to gather stats on loop
1463     uncommon_trap(Deoptimization::Reason_unreached,
1464                   Deoptimization::Action_reinterpret,
1465                   nullptr, "cold");
1466     if (C->eliminate_boxing()) {
1467       // Mark the successor blocks as parsed
1468       branch_block->next_path_num();

1519   }
1520 
1521   // Generate real control flow
1522   float true_prob = (taken_if_true ? prob : untaken_prob);
1523   IfNode* iff = create_and_map_if(control(), tst, true_prob, cnt);
1524   assert(iff->_prob > 0.0f,"Optimizer made bad probability in parser");
1525   Node* taken_branch   = new IfTrueNode(iff);
1526   Node* untaken_branch = new IfFalseNode(iff);
1527   if (!taken_if_true) {  // Finish conversion to canonical form
1528     Node* tmp      = taken_branch;
1529     taken_branch   = untaken_branch;
1530     untaken_branch = tmp;
1531   }
1532 
1533   // Branch is taken:
1534   { PreserveJVMState pjvms(this);
1535     taken_branch = _gvn.transform(taken_branch);
1536     set_control(taken_branch);
1537 
1538     if (stopped()) {
1539       if (C->eliminate_boxing()) {
1540         // Mark the successor block as parsed
1541         branch_block->next_path_num();
1542       }
1543     } else {
1544       adjust_map_after_if(taken_btest, c, prob, branch_block);
1545       if (!stopped()) {
1546         merge(target_bci);








1547       }
1548     }
1549   }
1550 
1551   untaken_branch = _gvn.transform(untaken_branch);
1552   set_control(untaken_branch);
1553 
1554   // Branch not taken.
1555   if (stopped()) {
1556     if (C->eliminate_boxing()) {
1557       // Mark the successor block as parsed
1558       next_block->next_path_num();
1559     }
1560   } else {
1561     adjust_map_after_if(untaken_btest, c, untaken_prob, next_block);
1562   }
1563 
1564   if (do_stress_trap) {
1565     stress_trap(iff, counter, incr_store);
1566   }
1567 }
1568 




















































































































































































































































































































































































































1569 // Force unstable if traps to be taken randomly to trigger intermittent bugs such as incorrect debug information.
1570 // Add another if before the unstable if that checks a "random" condition at runtime (a simple shared counter) and
1571 // then either takes the trap or executes the original, unstable if.
1572 void Parse::stress_trap(IfNode* orig_iff, Node* counter, Node* incr_store) {
1573   // Search for an unstable if trap
1574   CallStaticJavaNode* trap = nullptr;
1575   assert(orig_iff->Opcode() == Op_If && orig_iff->outcnt() == 2, "malformed if");
1576   ProjNode* trap_proj = orig_iff->uncommon_trap_proj(trap, Deoptimization::Reason_unstable_if);
1577   if (trap == nullptr || !trap->jvms()->should_reexecute()) {
1578     // No suitable trap found. Remove unused counter load and increment.
1579     C->gvn_replace_by(incr_store, incr_store->in(MemNode::Memory));
1580     return;
1581   }
1582 
1583   // Remove trap from optimization list since we add another path to the trap.
1584   bool success = C->remove_unstable_if_trap(trap, true);
1585   assert(success, "Trap already modified");
1586 
1587   // Add a check before the original if that will trap with a certain frequency and execute the original if otherwise
1588   int freq_log = (C->random() % 31) + 1; // Random logarithmic frequency in [1, 31]

1621 }
1622 
1623 void Parse::maybe_add_predicate_after_if(Block* path) {
1624   if (path->is_SEL_head() && path->preds_parsed() == 0) {
1625     // Add predicates at bci of if dominating the loop so traps can be
1626     // recorded on the if's profile data
1627     int bc_depth = repush_if_args();
1628     add_parse_predicates();
1629     dec_sp(bc_depth);
1630     path->set_has_predicates();
1631   }
1632 }
1633 
1634 
1635 //----------------------------adjust_map_after_if------------------------------
1636 // Adjust the JVM state to reflect the result of taking this path.
1637 // Basically, it means inspecting the CmpNode controlling this
1638 // branch, seeing how it constrains a tested value, and then
1639 // deciding if it's worth our while to encode this constraint
1640 // as graph nodes in the current abstract interpretation map.
1641 void Parse::adjust_map_after_if(BoolTest::mask btest, Node* c, float prob, Block* path) {
1642   if (!c->is_Cmp()) {
1643     maybe_add_predicate_after_if(path);
1644     return;
1645   }
1646 
1647   if (stopped() || btest == BoolTest::illegal) {
1648     return;                             // nothing to do
1649   }
1650 
1651   bool is_fallthrough = (path == successor_for_bci(iter().next_bci()));
1652 
1653   if (path_is_suitable_for_uncommon_trap(prob)) {
1654     repush_if_args();
1655     Node* call = uncommon_trap(Deoptimization::Reason_unstable_if,
1656                   Deoptimization::Action_reinterpret,
1657                   nullptr,
1658                   (is_fallthrough ? "taken always" : "taken never"));
1659 
1660     if (call != nullptr) {
1661       C->record_unstable_if_trap(new UnstableIfTrap(call->as_CallStaticJava(), path));
1662     }
1663     return;
1664   }
1665 
1666   Node* val = c->in(1);
1667   Node* con = c->in(2);
1668   const Type* tcon = _gvn.type(con);
1669   const Type* tval = _gvn.type(val);
1670   bool have_con = tcon->singleton();
1671   if (tval->singleton()) {
1672     if (!have_con) {
1673       // Swap, so constant is in con.

1730     if (obj != nullptr && (con_type->isa_instptr() || con_type->isa_aryptr())) {
1731        // Found:
1732        //   Bool(CmpP(LoadKlass(obj._klass), ConP(Foo.klass)), [eq])
1733        // or the narrowOop equivalent.
1734        const Type* obj_type = _gvn.type(obj);
1735        const TypeOopPtr* tboth = obj_type->join_speculative(con_type)->isa_oopptr();
1736        if (tboth != nullptr && tboth->klass_is_exact() && tboth != obj_type &&
1737            tboth->higher_equal(obj_type)) {
1738           // obj has to be of the exact type Foo if the CmpP succeeds.
1739           int obj_in_map = map()->find_edge(obj);
1740           JVMState* jvms = this->jvms();
1741           if (obj_in_map >= 0 &&
1742               (jvms->is_loc(obj_in_map) || jvms->is_stk(obj_in_map))) {
1743             TypeNode* ccast = new CheckCastPPNode(control(), obj, tboth);
1744             const Type* tcc = ccast->as_Type()->type();
1745             assert(tcc != obj_type && tcc->higher_equal(obj_type), "must improve");
1746             // Delay transform() call to allow recovery of pre-cast value
1747             // at the control merge.
1748             _gvn.set_type_bottom(ccast);
1749             record_for_igvn(ccast);



1750             // Here's the payoff.
1751             replace_in_map(obj, ccast);
1752           }
1753        }
1754     }
1755   }
1756 
1757   int val_in_map = map()->find_edge(val);
1758   if (val_in_map < 0)  return;          // replace_in_map would be useless
1759   {
1760     JVMState* jvms = this->jvms();
1761     if (!(jvms->is_loc(val_in_map) ||
1762           jvms->is_stk(val_in_map)))
1763       return;                           // again, it would be useless
1764   }
1765 
1766   // Check for a comparison to a constant, and "know" that the compared
1767   // value is constrained on this path.
1768   assert(tcon->singleton(), "");
1769   ConstraintCastNode* ccast = nullptr;

1834   if (c->Opcode() == Op_CmpP &&
1835       (c->in(1)->Opcode() == Op_LoadKlass || c->in(1)->Opcode() == Op_DecodeNKlass) &&
1836       c->in(2)->is_Con()) {
1837     Node* load_klass = nullptr;
1838     Node* decode = nullptr;
1839     if (c->in(1)->Opcode() == Op_DecodeNKlass) {
1840       decode = c->in(1);
1841       load_klass = c->in(1)->in(1);
1842     } else {
1843       load_klass = c->in(1);
1844     }
1845     if (load_klass->in(2)->is_AddP()) {
1846       Node* addp = load_klass->in(2);
1847       Node* obj = addp->in(AddPNode::Address);
1848       const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr();
1849       if (obj_type->speculative_type_not_null() != nullptr) {
1850         ciKlass* k = obj_type->speculative_type();
1851         inc_sp(2);
1852         obj = maybe_cast_profiled_obj(obj, k);
1853         dec_sp(2);




1854         // Make the CmpP use the casted obj
1855         addp = basic_plus_adr(obj, addp->in(AddPNode::Offset));
1856         load_klass = load_klass->clone();
1857         load_klass->set_req(2, addp);
1858         load_klass = _gvn.transform(load_klass);
1859         if (decode != nullptr) {
1860           decode = decode->clone();
1861           decode->set_req(1, load_klass);
1862           load_klass = _gvn.transform(decode);
1863         }
1864         c = c->clone();
1865         c->set_req(1, load_klass);
1866         c = _gvn.transform(c);
1867       }
1868     }
1869   }
1870   return c;
1871 }
1872 
1873 //------------------------------do_one_bytecode--------------------------------

2573 
2574   case Bytecodes::_i2d:
2575     a = pop();
2576     b = _gvn.transform( new ConvI2DNode(a));
2577     push_pair(b);
2578     break;
2579 
2580   case Bytecodes::_iinc:        // Increment local
2581     i = iter().get_index();     // Get local index
2582     set_local( i, _gvn.transform( new AddINode( _gvn.intcon(iter().get_iinc_con()), local(i) ) ) );
2583     break;
2584 
2585   // Exit points of synchronized methods must have an unlock node
2586   case Bytecodes::_return:
2587     return_current(nullptr);
2588     break;
2589 
2590   case Bytecodes::_ireturn:
2591   case Bytecodes::_areturn:
2592   case Bytecodes::_freturn:
2593     return_current(pop());
2594     break;
2595   case Bytecodes::_lreturn:
2596     return_current(pop_pair());
2597     break;
2598   case Bytecodes::_dreturn:
2599     return_current(pop_pair());
2600     break;
2601 
2602   case Bytecodes::_athrow:
2603     // null exception oop throws null pointer exception
2604     null_check(peek());
2605     if (stopped())  return;
2606     // Hook the thrown exception directly to subsequent handlers.
2607     if (BailoutToInterpreterForThrows) {
2608       // Keep method interpreted from now on.
2609       uncommon_trap(Deoptimization::Reason_unhandled,
2610                     Deoptimization::Action_make_not_compilable);
2611       return;
2612     }
2613     if (env()->jvmti_can_post_on_exceptions()) {
2614       // check if we must post exception events, take uncommon trap if so (with must_throw = false)
2615       uncommon_trap_if_should_post_on_exceptions(Deoptimization::Reason_unhandled, false);
2616     }
2617     // Here if either can_post_on_exceptions or should_post_on_exceptions is false

2630 
2631     // See if we can get some profile data and hand it off to the next block
2632     Block *target_block = block()->successor_for_bci(target_bci);
2633     if (target_block->pred_count() != 1)  break;
2634     ciMethodData* methodData = method()->method_data();
2635     if (!methodData->is_mature())  break;
2636     ciProfileData* data = methodData->bci_to_data(bci());
2637     assert(data != nullptr && data->is_JumpData(), "need JumpData for taken branch");
2638     int taken = ((ciJumpData*)data)->taken();
2639     taken = method()->scale_count(taken);
2640     target_block->set_count(taken);
2641     break;
2642   }
2643 
2644   case Bytecodes::_ifnull:    btest = BoolTest::eq; goto handle_if_null;
2645   case Bytecodes::_ifnonnull: btest = BoolTest::ne; goto handle_if_null;
2646   handle_if_null:
2647     // If this is a backwards branch in the bytecodes, add Safepoint
2648     maybe_add_safepoint(iter().get_dest());
2649     a = null();
2650     b = pop();
2651     if (!_gvn.type(b)->speculative_maybe_null() &&
2652         !too_many_traps(Deoptimization::Reason_speculate_null_check)) {
2653       inc_sp(1);
2654       Node* null_ctl = top();
2655       b = null_check_oop(b, &null_ctl, true, true, true);
2656       assert(null_ctl->is_top(), "no null control here");
2657       dec_sp(1);
2658     } else if (_gvn.type(b)->speculative_always_null() &&
2659                !too_many_traps(Deoptimization::Reason_speculate_null_assert)) {
2660       inc_sp(1);
2661       b = null_assert(b);
2662       dec_sp(1);
2663     }
2664     c = _gvn.transform( new CmpPNode(b, a) );






2665     do_ifnull(btest, c);
2666     break;
2667 
2668   case Bytecodes::_if_acmpeq: btest = BoolTest::eq; goto handle_if_acmp;
2669   case Bytecodes::_if_acmpne: btest = BoolTest::ne; goto handle_if_acmp;
2670   handle_if_acmp:
2671     // If this is a backwards branch in the bytecodes, add Safepoint
2672     maybe_add_safepoint(iter().get_dest());
2673     a = pop();
2674     b = pop();
2675     c = _gvn.transform( new CmpPNode(b, a) );
2676     c = optimize_cmp_with_klass(c);
2677     do_if(btest, c);
2678     break;
2679 
2680   case Bytecodes::_ifeq: btest = BoolTest::eq; goto handle_ifxx;
2681   case Bytecodes::_ifne: btest = BoolTest::ne; goto handle_ifxx;
2682   case Bytecodes::_iflt: btest = BoolTest::lt; goto handle_ifxx;
2683   case Bytecodes::_ifle: btest = BoolTest::le; goto handle_ifxx;
2684   case Bytecodes::_ifgt: btest = BoolTest::gt; goto handle_ifxx;
2685   case Bytecodes::_ifge: btest = BoolTest::ge; goto handle_ifxx;
2686   handle_ifxx:
2687     // If this is a backwards branch in the bytecodes, add Safepoint
2688     maybe_add_safepoint(iter().get_dest());
2689     a = _gvn.intcon(0);
2690     b = pop();
2691     c = _gvn.transform( new CmpINode(b, a) );
2692     do_if(btest, c);
2693     break;
2694 
2695   case Bytecodes::_if_icmpeq: btest = BoolTest::eq; goto handle_if_icmp;
2696   case Bytecodes::_if_icmpne: btest = BoolTest::ne; goto handle_if_icmp;
2697   case Bytecodes::_if_icmplt: btest = BoolTest::lt; goto handle_if_icmp;

2712     break;
2713 
2714   case Bytecodes::_lookupswitch:
2715     do_lookupswitch();
2716     break;
2717 
2718   case Bytecodes::_invokestatic:
2719   case Bytecodes::_invokedynamic:
2720   case Bytecodes::_invokespecial:
2721   case Bytecodes::_invokevirtual:
2722   case Bytecodes::_invokeinterface:
2723     do_call();
2724     break;
2725   case Bytecodes::_checkcast:
2726     do_checkcast();
2727     break;
2728   case Bytecodes::_instanceof:
2729     do_instanceof();
2730     break;
2731   case Bytecodes::_anewarray:
2732     do_anewarray();
2733     break;
2734   case Bytecodes::_newarray:
2735     do_newarray((BasicType)iter().get_index());
2736     break;
2737   case Bytecodes::_multianewarray:
2738     do_multianewarray();
2739     break;
2740   case Bytecodes::_new:
2741     do_new();
2742     break;
2743 
2744   case Bytecodes::_jsr:
2745   case Bytecodes::_jsr_w:
2746     do_jsr();
2747     break;
2748 
2749   case Bytecodes::_ret:
2750     do_ret();
2751     break;
2752 

2762   case Bytecodes::_breakpoint:
2763     // Breakpoint set concurrently to compile
2764     // %%% use an uncommon trap?
2765     C->record_failure("breakpoint in method");
2766     return;
2767 
2768   default:
2769 #ifndef PRODUCT
2770     map()->dump(99);
2771 #endif
2772     tty->print("\nUnhandled bytecode %s\n", Bytecodes::name(bc()) );
2773     ShouldNotReachHere();
2774   }
2775 
2776 #ifndef PRODUCT
2777   if (failing()) { return; }
2778   constexpr int perBytecode = 6;
2779   if (C->should_print_igv(perBytecode)) {
2780     IdealGraphPrinter* printer = C->igv_printer();
2781     char buffer[256];
2782     jio_snprintf(buffer, sizeof(buffer), "Bytecode %d: %s", bci(), Bytecodes::name(bc()));
2783     bool old = printer->traverse_outs();
2784     printer->set_traverse_outs(true);
2785     printer->print_graph(buffer);
2786     printer->set_traverse_outs(old);
2787   }
2788 #endif
2789 }

   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "ci/ciInlineKlass.hpp"
  26 #include "ci/ciMethodData.hpp"
  27 #include "ci/ciSymbols.hpp"
  28 #include "classfile/vmSymbols.hpp"
  29 #include "compiler/compileLog.hpp"
  30 #include "interpreter/linkResolver.hpp"
  31 #include "jvm_io.h"
  32 #include "memory/resourceArea.hpp"
  33 #include "memory/universe.hpp"
  34 #include "oops/oop.inline.hpp"
  35 #include "opto/addnode.hpp"
  36 #include "opto/castnode.hpp"
  37 #include "opto/convertnode.hpp"
  38 #include "opto/divnode.hpp"
  39 #include "opto/idealGraphPrinter.hpp"
  40 #include "opto/idealKit.hpp"
  41 #include "opto/inlinetypenode.hpp"
  42 #include "opto/matcher.hpp"
  43 #include "opto/memnode.hpp"
  44 #include "opto/mulnode.hpp"
  45 #include "opto/opaquenode.hpp"
  46 #include "opto/parse.hpp"
  47 #include "opto/runtime.hpp"
  48 #include "runtime/deoptimization.hpp"
  49 #include "runtime/sharedRuntime.hpp"
  50 
  51 #ifndef PRODUCT
  52 extern uint explicit_null_checks_inserted,
  53             explicit_null_checks_elided;
  54 #endif
  55 
  56 Node* Parse::record_profile_for_speculation_at_array_load(Node* ld) {
  57   // Feed unused profile data to type speculation
  58   if (UseTypeSpeculation && UseArrayLoadStoreProfile) {
  59     ciKlass* array_type = nullptr;
  60     ciKlass* element_type = nullptr;
  61     ProfilePtrKind element_ptr = ProfileMaybeNull;
  62     bool flat_array = true;
  63     bool null_free_array = true;
  64     method()->array_access_profiled_type(bci(), array_type, element_type, element_ptr, flat_array, null_free_array);
  65     if (element_type != nullptr || element_ptr != ProfileMaybeNull) {
  66       ld = record_profile_for_speculation(ld, element_type, element_ptr);
  67     }
  68   }
  69   return ld;
  70 }
  71 
  72 
  73 //---------------------------------array_load----------------------------------
  74 void Parse::array_load(BasicType bt) {
  75   const Type* elemtype = Type::TOP;

  76   Node* adr = array_addressing(bt, 0, elemtype);
  77   if (stopped())  return;     // guaranteed null or range check
  78 
  79   Node* array_index = pop();
  80   Node* array = pop();
  81 
  82   // Handle inline type arrays
  83   const TypeOopPtr* element_ptr = elemtype->make_oopptr();
  84   const TypeAryPtr* array_type = _gvn.type(array)->is_aryptr();
  85 
  86   if (!array_type->is_not_flat()) {
  87     // Cannot statically determine if array is a flat array, emit runtime check
  88     assert(UseArrayFlattening && is_reference_type(bt) && element_ptr->can_be_inline_type() &&
  89            (!element_ptr->is_inlinetypeptr() || element_ptr->inline_klass()->maybe_flat_in_array()), "array can't be flat");
  90     IdealKit ideal(this);
  91     IdealVariable res(ideal);
  92     ideal.declarations_done();
  93     ideal.if_then(flat_array_test(array, /* flat = */ false)); {
  94       // Non-flat array
  95       sync_kit(ideal);
  96       if (!array_type->is_flat()) {
  97         assert(array_type->is_flat() || control()->in(0)->as_If()->is_flat_array_check(&_gvn), "Should be found");
  98         const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt);
  99         DecoratorSet decorator_set = IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD;
 100         if (needs_range_check(array_type->size(), array_index)) {
 101           // We've emitted a RangeCheck but now insert an additional check between the range check and the actual load.
 102           // We cannot pin the load to two separate nodes. Instead, we pin it conservatively here such that it cannot
 103           // possibly float above the range check at any point.
 104           decorator_set |= C2_UNKNOWN_CONTROL_LOAD;
 105         }
 106         Node* ld = access_load_at(array, adr, adr_type, element_ptr, bt, decorator_set);
 107         if (element_ptr->is_inlinetypeptr()) {
 108           ld = InlineTypeNode::make_from_oop(this, ld, element_ptr->inline_klass());
 109         }
 110         ideal.set(res, ld);
 111       }
 112       ideal.sync_kit(this);
 113     } ideal.else_(); {
 114       // Flat array
 115       sync_kit(ideal);
 116       if (!array_type->is_not_flat()) {
 117         if (element_ptr->is_inlinetypeptr()) {
 118           ciInlineKlass* vk = element_ptr->inline_klass();
 119           Node* flat_array = cast_to_flat_array(array, vk, false, false, false);
 120           Node* vt = InlineTypeNode::make_from_flat_array(this, vk, flat_array, array_index);
 121           ideal.set(res, vt);
 122         } else {
 123           // Element type is unknown, and thus we cannot statically determine the exact flat array layout. Emit a
 124           // runtime call to correctly load the inline type element from the flat array.
 125           Node* inline_type = load_from_unknown_flat_array(array, array_index, element_ptr);
 126           bool is_null_free = array_type->is_null_free() || !UseNullableValueFlattening;
 127           if (is_null_free) {
 128             inline_type = cast_not_null(inline_type);
 129           }
 130           ideal.set(res, inline_type);
 131         }
 132       }
 133       ideal.sync_kit(this);
 134     } ideal.end_if();
 135     sync_kit(ideal);
 136     Node* ld = _gvn.transform(ideal.value(res));
 137     ld = record_profile_for_speculation_at_array_load(ld);
 138     push_node(bt, ld);
 139     return;
 140   }
 141 
 142   if (elemtype == TypeInt::BOOL) {
 143     bt = T_BOOLEAN;
 144   }
 145   const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt);

 146   Node* ld = access_load_at(array, adr, adr_type, elemtype, bt,
 147                             IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD);
 148   ld = record_profile_for_speculation_at_array_load(ld);
 149   // Loading an inline type from a non-flat array
 150   if (element_ptr != nullptr && element_ptr->is_inlinetypeptr()) {
 151     assert(!array_type->is_null_free() || !element_ptr->maybe_null(), "inline type array elements should never be null");
 152     ld = InlineTypeNode::make_from_oop(this, ld, element_ptr->inline_klass());
 153   }
 154   push_node(bt, ld);
 155 }
 156 
 157 Node* Parse::load_from_unknown_flat_array(Node* array, Node* array_index, const TypeOopPtr* element_ptr) {
 158   // Below membars keep this access to an unknown flat array correctly
 159   // ordered with other unknown and known flat array accesses.
 160   insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::INLINES));
 161 
 162   Node* call = nullptr;
 163   {
 164     // Re-execute flat array load if runtime call triggers deoptimization
 165     PreserveReexecuteState preexecs(this);
 166     jvms()->set_bci(_bci);
 167     jvms()->set_should_reexecute(true);
 168     inc_sp(2);
 169     kill_dead_locals();
 170     call = make_runtime_call(RC_NO_LEAF | RC_NO_IO,
 171                              OptoRuntime::load_unknown_inline_Type(),
 172                              OptoRuntime::load_unknown_inline_Java(),
 173                              nullptr, TypeRawPtr::BOTTOM,
 174                              array, array_index);
 175   }
 176   make_slow_call_ex(call, env()->Throwable_klass(), false);
 177   Node* buffer = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
 178 
 179   insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::INLINES));
 180 
 181   // Keep track of the information that the inline type is in flat arrays
 182   const Type* unknown_value = element_ptr->is_instptr()->cast_to_flat_in_array();
 183   return _gvn.transform(new CheckCastPPNode(control(), buffer, unknown_value));
 184 }
 185 
 186 //--------------------------------array_store----------------------------------
 187 void Parse::array_store(BasicType bt) {
 188   const Type* elemtype = Type::TOP;
 189   Node* adr = array_addressing(bt, type2size[bt], elemtype);

 190   if (stopped())  return;     // guaranteed null or range check
 191   Node* stored_value_casted = nullptr;
 192   if (bt == T_OBJECT) {
 193     stored_value_casted = array_store_check(adr, elemtype);
 194     if (stopped()) {
 195       return;
 196     }
 197   }
 198   Node* const stored_value = pop_node(bt); // Value to store
 199   Node* const array_index = pop();         // Index in the array
 200   Node* array = pop();                     // The array itself
 201 
 202   const TypeAryPtr* array_type = _gvn.type(array)->is_aryptr();
 203   const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt);


 204 
 205   if (elemtype == TypeInt::BOOL) {
 206     bt = T_BOOLEAN;
 207   } else if (bt == T_OBJECT) {
 208     elemtype = elemtype->make_oopptr();
 209     const Type* stored_value_casted_type = _gvn.type(stored_value_casted);
 210     // Based on the value to be stored, try to determine if the array is not null-free and/or not flat.
 211     // This is only legal for non-null stores because the array_store_check always passes for null, even
 212     // if the array is null-free. Null stores are handled in GraphKit::inline_array_null_guard().
 213     bool not_inline = !stored_value_casted_type->maybe_null() && !stored_value_casted_type->is_oopptr()->can_be_inline_type();
 214     bool not_null_free = not_inline;
 215     bool not_flat = not_inline || ( stored_value_casted_type->is_inlinetypeptr() &&
 216                                    !stored_value_casted_type->inline_klass()->maybe_flat_in_array());
 217     if (!array_type->is_not_null_free() && not_null_free) {
 218       // Storing a non-inline type, mark array as not null-free.
 219       array_type = array_type->cast_to_not_null_free();
 220       Node* cast = _gvn.transform(new CheckCastPPNode(control(), array, array_type));
 221       replace_in_map(array, cast);
 222       array = cast;
 223     }
 224     if (!array_type->is_not_flat() && not_flat) {
 225       // Storing to a non-flat array, mark array as not flat.
 226       array_type = array_type->cast_to_not_flat();
 227       Node* cast = _gvn.transform(new CheckCastPPNode(control(), array, array_type));
 228       replace_in_map(array, cast);
 229       array = cast;
 230     }
 231 
 232     if (!array_type->is_flat() && array_type->is_null_free()) {
 233       // Store to non-flat null-free inline type array (elements can never be null)
 234       assert(!stored_value_casted_type->maybe_null(), "should be guaranteed by array store check");
 235       if (elemtype->is_inlinetypeptr() && elemtype->inline_klass()->is_empty()) {
 236         // Ignore empty inline stores, array is already initialized.
 237         return;
 238       }
 239     } else if (!array_type->is_not_flat()) {
 240       // Array might be a flat array, emit runtime checks (for nullptr, a simple inline_array_null_guard is sufficient).
 241       assert(UseArrayFlattening && !not_flat && elemtype->is_oopptr()->can_be_inline_type() &&
 242              (!array_type->klass_is_exact() || array_type->is_flat()), "array can't be a flat array");
 243       // TODO 8350865 Depending on the available layouts, we can avoid this check in below flat/not-flat branches. Also the safe_for_replace arg is now always true.
 244       array = inline_array_null_guard(array, stored_value_casted, 3, true);
 245       IdealKit ideal(this);
 246       ideal.if_then(flat_array_test(array, /* flat = */ false)); {
 247         // Non-flat array
 248         if (!array_type->is_flat()) {
 249           sync_kit(ideal);
 250           assert(array_type->is_flat() || ideal.ctrl()->in(0)->as_If()->is_flat_array_check(&_gvn), "Should be found");
 251           inc_sp(3);
 252           access_store_at(array, adr, adr_type, stored_value_casted, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY, false);
 253           dec_sp(3);
 254           ideal.sync_kit(this);
 255         }
 256       } ideal.else_(); {
 257         // Flat array
 258         sync_kit(ideal);
 259         if (!array_type->is_not_flat()) {
 260           // Try to determine the inline klass type of the stored value
 261           ciInlineKlass* vk = nullptr;
 262           if (stored_value_casted_type->is_inlinetypeptr()) {
 263             vk = stored_value_casted_type->inline_klass();
 264           } else if (elemtype->is_inlinetypeptr()) {
 265             vk = elemtype->inline_klass();
 266           }
 267 
 268           if (vk != nullptr) {
 269             // Element type is known, cast and store to flat array layout.
 270             Node* flat_array = cast_to_flat_array(array, vk, false, false, false);
 271 
 272             // Re-execute flat array store if buffering triggers deoptimization
 273             PreserveReexecuteState preexecs(this);
 274             jvms()->set_should_reexecute(true);
 275             inc_sp(3);
 276 
 277             if (!stored_value_casted->is_InlineType()) {
 278               assert(_gvn.type(stored_value_casted) == TypePtr::NULL_PTR, "Unexpected value");
 279               stored_value_casted = InlineTypeNode::make_null(_gvn, vk);
 280             }
 281 
 282             stored_value_casted->as_InlineType()->store_flat_array(this, flat_array, array_index);
 283           } else {
 284             // Element type is unknown, emit a runtime call since the flat array layout is not statically known.
 285             store_to_unknown_flat_array(array, array_index, stored_value_casted);
 286           }
 287         }
 288         ideal.sync_kit(this);
 289       }
 290       ideal.end_if();
 291       sync_kit(ideal);
 292       return;
 293     } else if (!array_type->is_not_null_free()) {
 294       // Array is not flat but may be null free
 295       assert(elemtype->is_oopptr()->can_be_inline_type(), "array can't be null-free");
 296       array = inline_array_null_guard(array, stored_value_casted, 3, true);
 297     }
 298   }
 299   inc_sp(3);
 300   access_store_at(array, adr, adr_type, stored_value, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY);
 301   dec_sp(3);
 302 }
 303 
 304 // Emit a runtime call to store to a flat array whose element type is either unknown (i.e. we do not know the flat
 305 // array layout) or not exact (could have different flat array layouts at runtime).
 306 void Parse::store_to_unknown_flat_array(Node* array, Node* const idx, Node* non_null_stored_value) {
 307   // Below membars keep this access to an unknown flat array correctly
 308   // ordered with other unknown and known flat array accesses.
 309   insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::INLINES));
 310 
 311   Node* call = nullptr;
 312   {
 313     // Re-execute flat array store if runtime call triggers deoptimization
 314     PreserveReexecuteState preexecs(this);
 315     jvms()->set_bci(_bci);
 316     jvms()->set_should_reexecute(true);
 317     inc_sp(3);
 318     kill_dead_locals();
 319     call = make_runtime_call(RC_NO_LEAF | RC_NO_IO,
 320                       OptoRuntime::store_unknown_inline_Type(),
 321                       OptoRuntime::store_unknown_inline_Java(),
 322                       nullptr, TypeRawPtr::BOTTOM,
 323                       non_null_stored_value, array, idx);
 324   }
 325   make_slow_call_ex(call, env()->Throwable_klass(), false);
 326 
 327   insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::INLINES));
 328 }
 329 
 330 //------------------------------array_addressing-------------------------------
 331 // Pull array and index from the stack.  Compute pointer-to-element.
 332 Node* Parse::array_addressing(BasicType type, int vals, const Type*& elemtype) {
 333   Node *idx   = peek(0+vals);   // Get from stack without popping
 334   Node *ary   = peek(1+vals);   // in case of exception
 335 
 336   // Null check the array base, with correct stack contents
 337   ary = null_check(ary, T_ARRAY);
 338   // Compile-time detect of null-exception?
 339   if (stopped())  return top();
 340 
 341   const TypeAryPtr* arytype  = _gvn.type(ary)->is_aryptr();
 342   const TypeInt*    sizetype = arytype->size();
 343   elemtype = arytype->elem();
 344 
 345   if (UseUniqueSubclasses) {
 346     const Type* el = elemtype->make_ptr();
 347     if (el && el->isa_instptr()) {
 348       const TypeInstPtr* toop = el->is_instptr();
 349       if (toop->instance_klass()->unique_concrete_subklass()) {
 350         // If we load from "AbstractClass[]" we must see "ConcreteSubClass".
 351         const Type* subklass = Type::get_const_type(toop->instance_klass());
 352         elemtype = subklass->join_speculative(el);
 353       }
 354     }
 355   }
 356 











 357   if (!arytype->is_loaded()) {
 358     // Only fails for some -Xcomp runs
 359     // The class is unloaded.  We have to run this bytecode in the interpreter.
 360     ciKlass* klass = arytype->unloaded_klass();
 361 
 362     uncommon_trap(Deoptimization::Reason_unloaded,
 363                   Deoptimization::Action_reinterpret,
 364                   klass, "!loaded array");
 365     return top();
 366   }
 367 
 368   ary = create_speculative_inline_type_array_checks(ary, arytype, elemtype);











 369 
 370   if (needs_range_check(sizetype, idx)) {
 371     create_range_check(idx, ary, sizetype);
 372   } else if (C->log() != nullptr) {
 373     C->log()->elem("observe that='!need_range_check'");



























 374   }
 375 
 376   // Check for always knowing you are throwing a range-check exception
 377   if (stopped())  return top();
 378 
 379   // Make array address computation control dependent to prevent it
 380   // from floating above the range check during loop optimizations.
 381   Node* ptr = array_element_address(ary, idx, type, sizetype, control());
 382   assert(ptr != top(), "top should go hand-in-hand with stopped");
 383 
 384   return ptr;
 385 }
 386 
 387 // Check if we need a range check for an array access. This is the case if the index is either negative or if it could
 388 // be greater or equal the smallest possible array size (i.e. out-of-bounds).
 389 bool Parse::needs_range_check(const TypeInt* size_type, const Node* index) const {
 390   const TypeInt* index_type = _gvn.type(index)->is_int();
 391   return index_type->_hi >= size_type->_lo || index_type->_lo < 0;
 392 }
 393 
 394 void Parse::create_range_check(Node* idx, Node* ary, const TypeInt* sizetype) {
 395   Node* tst;
 396   if (sizetype->_hi <= 0) {
 397     // The greatest array bound is negative, so we can conclude that we're
 398     // compiling unreachable code, but the unsigned compare trick used below
 399     // only works with non-negative lengths.  Instead, hack "tst" to be zero so
 400     // the uncommon_trap path will always be taken.
 401     tst = _gvn.intcon(0);
 402   } else {
 403     // Range is constant in array-oop, so we can use the original state of mem
 404     Node* len = load_array_length(ary);
 405 
 406     // Test length vs index (standard trick using unsigned compare)
 407     Node* chk = _gvn.transform(new CmpUNode(idx, len) );
 408     BoolTest::mask btest = BoolTest::lt;
 409     tst = _gvn.transform(new BoolNode(chk, btest) );
 410   }
 411   RangeCheckNode* rc = new RangeCheckNode(control(), tst, PROB_MAX, COUNT_UNKNOWN);
 412   _gvn.set_type(rc, rc->Value(&_gvn));
 413   if (!tst->is_Con()) {
 414     record_for_igvn(rc);
 415   }
 416   set_control(_gvn.transform(new IfTrueNode(rc)));
 417   // Branch to failure if out of bounds
 418   {
 419     PreserveJVMState pjvms(this);
 420     set_control(_gvn.transform(new IfFalseNode(rc)));
 421     if (C->allow_range_check_smearing()) {
 422       // Do not use builtin_throw, since range checks are sometimes
 423       // made more stringent by an optimistic transformation.
 424       // This creates "tentative" range checks at this point,
 425       // which are not guaranteed to throw exceptions.
 426       // See IfNode::Ideal, is_range_check, adjust_check.
 427       uncommon_trap(Deoptimization::Reason_range_check,
 428                     Deoptimization::Action_make_not_entrant,
 429                     nullptr, "range_check");
 430     } else {
 431       // If we have already recompiled with the range-check-widening
 432       // heroic optimization turned off, then we must really be throwing
 433       // range check exceptions.
 434       builtin_throw(Deoptimization::Reason_range_check);
 435     }
 436   }
 437 }
 438 
 439 // For inline type arrays, we can use the profiling information for array accesses to speculate on the type, flatness,
 440 // and null-freeness. We can either prepare the speculative type for later uses or emit explicit speculative checks with
 441 // traps now. In the latter case, the speculative type guarantees can avoid additional runtime checks later (e.g.
 442 // non-null-free implies non-flat which allows us to remove flatness checks). This makes the graph simpler.
 443 Node* Parse::create_speculative_inline_type_array_checks(Node* array, const TypeAryPtr* array_type,
 444                                                          const Type*& element_type) {
 445   if (!array_type->is_flat() && !array_type->is_not_flat()) {
 446     // For arrays that might be flat, speculate that the array has the exact type reported in the profile data such that
 447     // we can rely on a fixed memory layout (i.e. either a flat layout or not).
 448     array = cast_to_speculative_array_type(array, array_type, element_type);
 449   } else if (UseTypeSpeculation && UseArrayLoadStoreProfile) {
 450     // Array is known to be either flat or not flat. If possible, update the speculative type by using the profile data
 451     // at this bci.
 452     array = cast_to_profiled_array_type(array);
 453   }
 454 
 455   // Even though the type does not tell us whether we have an inline type array or not, we can still check the profile data
 456   // whether we have a non-null-free or non-flat array. Speculating on a non-null-free array doesn't help aaload but could
 457   // be profitable for a subsequent aastore.
 458   if (!array_type->is_null_free() && !array_type->is_not_null_free()) {
 459     array = speculate_non_null_free_array(array, array_type);
 460   }
 461   if (!array_type->is_flat() && !array_type->is_not_flat()) {
 462     array = speculate_non_flat_array(array, array_type);
 463   }
 464   return array;
 465 }
 466 
 467 // Speculate that the array has the exact type reported in the profile data. We emit a trap when this turns out to be
 468 // wrong. On the fast path, we add a CheckCastPP to use the exact type.
 469 Node* Parse::cast_to_speculative_array_type(Node* const array, const TypeAryPtr*& array_type, const Type*& element_type) {
 470   Deoptimization::DeoptReason reason = Deoptimization::Reason_speculate_class_check;
 471   ciKlass* speculative_array_type = array_type->speculative_type();
 472   if (too_many_traps_or_recompiles(reason) || speculative_array_type == nullptr) {
 473     // No speculative type, check profile data at this bci
 474     speculative_array_type = nullptr;
 475     reason = Deoptimization::Reason_class_check;
 476     if (UseArrayLoadStoreProfile && !too_many_traps_or_recompiles(reason)) {
 477       ciKlass* profiled_element_type = nullptr;
 478       ProfilePtrKind element_ptr = ProfileMaybeNull;
 479       bool flat_array = true;
 480       bool null_free_array = true;
 481       method()->array_access_profiled_type(bci(), speculative_array_type, profiled_element_type, element_ptr, flat_array,
 482                                            null_free_array);
 483     }
 484   }
 485   if (speculative_array_type != nullptr) {
 486     // Speculate that this array has the exact type reported by profile data
 487     Node* casted_array = nullptr;
 488     DEBUG_ONLY(Node* old_control = control();)
 489     Node* slow_ctl = type_check_receiver(array, speculative_array_type, 1.0, &casted_array);
 490     if (stopped()) {
 491       // The check always fails and therefore profile information is incorrect. Don't use it.
 492       assert(old_control == slow_ctl, "type check should have been removed");
 493       set_control(slow_ctl);
 494     } else if (!slow_ctl->is_top()) {
 495       { PreserveJVMState pjvms(this);
 496         set_control(slow_ctl);
 497         uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
 498       }
 499       replace_in_map(array, casted_array);
 500       array_type = _gvn.type(casted_array)->is_aryptr();
 501       element_type = array_type->elem();
 502       return casted_array;
 503     }
 504   }
 505   return array;
 506 }
 507 
 508 // Create a CheckCastPP when the speculative type can improve the current type.
 509 Node* Parse::cast_to_profiled_array_type(Node* const array) {
 510   ciKlass* array_type = nullptr;
 511   ciKlass* element_type = nullptr;
 512   ProfilePtrKind element_ptr = ProfileMaybeNull;
 513   bool flat_array = true;
 514   bool null_free_array = true;
 515   method()->array_access_profiled_type(bci(), array_type, element_type, element_ptr, flat_array, null_free_array);
 516   if (array_type != nullptr) {
 517     return record_profile_for_speculation(array, array_type, ProfileMaybeNull);
 518   }
 519   return array;
 520 }
 521 
 522 // Speculate that the array is non-null-free. We emit a trap when this turns out to be
 523 // wrong. On the fast path, we add a CheckCastPP to use the non-null-free type.
 524 Node* Parse::speculate_non_null_free_array(Node* const array, const TypeAryPtr*& array_type) {
 525   bool null_free_array = true;
 526   Deoptimization::DeoptReason reason = Deoptimization::Reason_none;
 527   if (array_type->speculative() != nullptr &&
 528       array_type->speculative()->is_aryptr()->is_not_null_free() &&
 529       !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) {
 530     null_free_array = false;
 531     reason = Deoptimization::Reason_speculate_class_check;
 532   } else if (UseArrayLoadStoreProfile && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) {
 533     ciKlass* profiled_array_type = nullptr;
 534     ciKlass* profiled_element_type = nullptr;
 535     ProfilePtrKind element_ptr = ProfileMaybeNull;
 536     bool flat_array = true;
 537     method()->array_access_profiled_type(bci(), profiled_array_type, profiled_element_type, element_ptr, flat_array,
 538                                          null_free_array);
 539     reason = Deoptimization::Reason_class_check;
 540   }
 541   if (!null_free_array) {
 542     { // Deoptimize if null-free array
 543       BuildCutout unless(this, null_free_array_test(array, /* null_free = */ false), PROB_MAX);
 544       uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
 545     }
 546     assert(!stopped(), "null-free array should have been caught earlier");
 547     Node* casted_array = _gvn.transform(new CheckCastPPNode(control(), array, array_type->cast_to_not_null_free()));
 548     replace_in_map(array, casted_array);
 549     array_type = _gvn.type(casted_array)->is_aryptr();
 550     return casted_array;
 551   }
 552   return array;
 553 }
 554 
 555 // Speculate that the array is non-flat. We emit a trap when this turns out to be wrong.
 556 // On the fast path, we add a CheckCastPP to use the non-flat type.
 557 Node* Parse::speculate_non_flat_array(Node* const array, const TypeAryPtr* const array_type) {
 558   bool flat_array = true;
 559   Deoptimization::DeoptReason reason = Deoptimization::Reason_none;
 560   if (array_type->speculative() != nullptr &&
 561       array_type->speculative()->is_aryptr()->is_not_flat() &&
 562       !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) {
 563     flat_array = false;
 564     reason = Deoptimization::Reason_speculate_class_check;
 565   } else if (UseArrayLoadStoreProfile && !too_many_traps_or_recompiles(reason)) {
 566     ciKlass* profiled_array_type = nullptr;
 567     ciKlass* profiled_element_type = nullptr;
 568     ProfilePtrKind element_ptr = ProfileMaybeNull;
 569     bool null_free_array = true;
 570     method()->array_access_profiled_type(bci(), profiled_array_type, profiled_element_type, element_ptr, flat_array,
 571                                          null_free_array);
 572     reason = Deoptimization::Reason_class_check;
 573   }
 574   if (!flat_array) {
 575     { // Deoptimize if flat array
 576       BuildCutout unless(this, flat_array_test(array, /* flat = */ false), PROB_MAX);
 577       uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
 578     }
 579     assert(!stopped(), "flat array should have been caught earlier");
 580     Node* casted_array = _gvn.transform(new CheckCastPPNode(control(), array, array_type->cast_to_not_flat()));
 581     replace_in_map(array, casted_array);
 582     return casted_array;
 583   }
 584   return array;
 585 }
 586 
 587 // returns IfNode
 588 IfNode* Parse::jump_if_fork_int(Node* a, Node* b, BoolTest::mask mask, float prob, float cnt) {
 589   Node   *cmp = _gvn.transform(new CmpINode(a, b)); // two cases: shiftcount > 32 and shiftcount <= 32
 590   Node   *tst = _gvn.transform(new BoolNode(cmp, mask));
 591   IfNode *iff = create_and_map_if(control(), tst, prob, cnt);
 592   return iff;
 593 }
 594 
 595 
 596 // sentinel value for the target bci to mark never taken branches
 597 // (according to profiling)
 598 static const int never_reached = INT_MAX;
 599 
 600 //------------------------------helper for tableswitch-------------------------
 601 void Parse::jump_if_true_fork(IfNode *iff, int dest_bci_if_true, bool unc) {
 602   // True branch, use existing map info
 603   { PreserveJVMState pjvms(this);
 604     Node *iftrue  = _gvn.transform( new IfTrueNode (iff) );
 605     set_control( iftrue );

1802   // False branch
1803   Node* iffalse = _gvn.transform( new IfFalseNode(iff) );
1804   set_control(iffalse);
1805 
1806   if (stopped()) {              // Path is dead?
1807     NOT_PRODUCT(explicit_null_checks_elided++);
1808     if (C->eliminate_boxing()) {
1809       // Mark the successor block as parsed
1810       next_block->next_path_num();
1811     }
1812   } else  {                     // Path is live.
1813     adjust_map_after_if(BoolTest(btest).negate(), c, 1.0-prob, next_block);
1814   }
1815 
1816   if (do_stress_trap) {
1817     stress_trap(iff, counter, incr_store);
1818   }
1819 }
1820 
1821 //------------------------------------do_if------------------------------------
1822 void Parse::do_if(BoolTest::mask btest, Node* c, bool can_trap, bool new_path, Node** ctrl_taken) {
1823   int target_bci = iter().get_dest();
1824 
1825   Block* branch_block = successor_for_bci(target_bci);
1826   Block* next_block   = successor_for_bci(iter().next_bci());
1827 
1828   float cnt;
1829   float prob = branch_prediction(cnt, btest, target_bci, c);
1830   float untaken_prob = 1.0 - prob;
1831 
1832   if (prob == PROB_UNKNOWN) {
1833     if (PrintOpto && Verbose) {
1834       tty->print_cr("Never-taken edge stops compilation at bci %d", bci());
1835     }
1836     repush_if_args(); // to gather stats on loop
1837     uncommon_trap(Deoptimization::Reason_unreached,
1838                   Deoptimization::Action_reinterpret,
1839                   nullptr, "cold");
1840     if (C->eliminate_boxing()) {
1841       // Mark the successor blocks as parsed
1842       branch_block->next_path_num();

1893   }
1894 
1895   // Generate real control flow
1896   float true_prob = (taken_if_true ? prob : untaken_prob);
1897   IfNode* iff = create_and_map_if(control(), tst, true_prob, cnt);
1898   assert(iff->_prob > 0.0f,"Optimizer made bad probability in parser");
1899   Node* taken_branch   = new IfTrueNode(iff);
1900   Node* untaken_branch = new IfFalseNode(iff);
1901   if (!taken_if_true) {  // Finish conversion to canonical form
1902     Node* tmp      = taken_branch;
1903     taken_branch   = untaken_branch;
1904     untaken_branch = tmp;
1905   }
1906 
1907   // Branch is taken:
1908   { PreserveJVMState pjvms(this);
1909     taken_branch = _gvn.transform(taken_branch);
1910     set_control(taken_branch);
1911 
1912     if (stopped()) {
1913       if (C->eliminate_boxing() && !new_path) {
1914         // Mark the successor block as parsed (if we haven't created a new path)
1915         branch_block->next_path_num();
1916       }
1917     } else {
1918       adjust_map_after_if(taken_btest, c, prob, branch_block, can_trap);
1919       if (!stopped()) {
1920         if (new_path) {
1921           // Merge by using a new path
1922           merge_new_path(target_bci);
1923         } else if (ctrl_taken != nullptr) {
1924           // Don't merge but save taken branch to be wired by caller
1925           *ctrl_taken = control();
1926         } else {
1927           merge(target_bci);
1928         }
1929       }
1930     }
1931   }
1932 
1933   untaken_branch = _gvn.transform(untaken_branch);
1934   set_control(untaken_branch);
1935 
1936   // Branch not taken.
1937   if (stopped() && ctrl_taken == nullptr) {
1938     if (C->eliminate_boxing()) {
1939       // Mark the successor block as parsed (if caller does not re-wire control flow)
1940       next_block->next_path_num();
1941     }
1942   } else {
1943     adjust_map_after_if(untaken_btest, c, untaken_prob, next_block, can_trap);
1944   }
1945 
1946   if (do_stress_trap) {
1947     stress_trap(iff, counter, incr_store);
1948   }
1949 }
1950 
1951 
1952 static ProfilePtrKind speculative_ptr_kind(const TypeOopPtr* t) {
1953   if (t->speculative() == nullptr) {
1954     return ProfileUnknownNull;
1955   }
1956   if (t->speculative_always_null()) {
1957     return ProfileAlwaysNull;
1958   }
1959   if (t->speculative_maybe_null()) {
1960     return ProfileMaybeNull;
1961   }
1962   return ProfileNeverNull;
1963 }
1964 
1965 void Parse::acmp_always_null_input(Node* input, const TypeOopPtr* tinput, BoolTest::mask btest, Node* eq_region) {
1966   inc_sp(2);
1967   Node* cast = null_check_common(input, T_OBJECT, true, nullptr,
1968                                  !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_check) &&
1969                                  speculative_ptr_kind(tinput) == ProfileAlwaysNull);
1970   dec_sp(2);
1971   if (btest == BoolTest::ne) {
1972     {
1973       PreserveJVMState pjvms(this);
1974       replace_in_map(input, cast);
1975       int target_bci = iter().get_dest();
1976       merge(target_bci);
1977     }
1978     record_for_igvn(eq_region);
1979     set_control(_gvn.transform(eq_region));
1980   } else {
1981     replace_in_map(input, cast);
1982   }
1983 }
1984 
1985 Node* Parse::acmp_null_check(Node* input, const TypeOopPtr* tinput, ProfilePtrKind input_ptr, Node*& null_ctl) {
1986   inc_sp(2);
1987   null_ctl = top();
1988   Node* cast = null_check_oop(input, &null_ctl,
1989                               input_ptr == ProfileNeverNull || (input_ptr == ProfileUnknownNull && !too_many_traps_or_recompiles(Deoptimization::Reason_null_check)),
1990                               false,
1991                               speculative_ptr_kind(tinput) == ProfileNeverNull &&
1992                               !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_check));
1993   dec_sp(2);
1994   assert(!stopped(), "null input should have been caught earlier");
1995   return cast;
1996 }
1997 
1998 void Parse::acmp_known_non_inline_type_input(Node* input, const TypeOopPtr* tinput, ProfilePtrKind input_ptr, ciKlass* input_type, BoolTest::mask btest, Node* eq_region) {
1999   Node* ne_region = new RegionNode(1);
2000   Node* null_ctl;
2001   Node* cast = acmp_null_check(input, tinput, input_ptr, null_ctl);
2002   ne_region->add_req(null_ctl);
2003 
2004   Node* slow_ctl = type_check_receiver(cast, input_type, 1.0, &cast);
2005   {
2006     PreserveJVMState pjvms(this);
2007     inc_sp(2);
2008     set_control(slow_ctl);
2009     Deoptimization::DeoptReason reason;
2010     if (tinput->speculative_type() != nullptr && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) {
2011       reason = Deoptimization::Reason_speculate_class_check;
2012     } else {
2013       reason = Deoptimization::Reason_class_check;
2014     }
2015     uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
2016   }
2017   ne_region->add_req(control());
2018 
2019   record_for_igvn(ne_region);
2020   set_control(_gvn.transform(ne_region));
2021   if (btest == BoolTest::ne) {
2022     {
2023       PreserveJVMState pjvms(this);
2024       if (null_ctl == top()) {
2025         replace_in_map(input, cast);
2026       }
2027       int target_bci = iter().get_dest();
2028       merge(target_bci);
2029     }
2030     record_for_igvn(eq_region);
2031     set_control(_gvn.transform(eq_region));
2032   } else {
2033     if (null_ctl == top()) {
2034       replace_in_map(input, cast);
2035     }
2036     set_control(_gvn.transform(ne_region));
2037   }
2038 }
2039 
2040 void Parse::acmp_unknown_non_inline_type_input(Node* input, const TypeOopPtr* tinput, ProfilePtrKind input_ptr, BoolTest::mask btest, Node* eq_region) {
2041   Node* ne_region = new RegionNode(1);
2042   Node* null_ctl;
2043   Node* cast = acmp_null_check(input, tinput, input_ptr, null_ctl);
2044   ne_region->add_req(null_ctl);
2045 
2046   {
2047     BuildCutout unless(this, inline_type_test(cast, /* is_inline = */ false), PROB_MAX);
2048     inc_sp(2);
2049     uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile);
2050   }
2051 
2052   ne_region->add_req(control());
2053 
2054   record_for_igvn(ne_region);
2055   set_control(_gvn.transform(ne_region));
2056   if (btest == BoolTest::ne) {
2057     {
2058       PreserveJVMState pjvms(this);
2059       if (null_ctl == top()) {
2060         replace_in_map(input, cast);
2061       }
2062       int target_bci = iter().get_dest();
2063       merge(target_bci);
2064     }
2065     record_for_igvn(eq_region);
2066     set_control(_gvn.transform(eq_region));
2067   } else {
2068     if (null_ctl == top()) {
2069       replace_in_map(input, cast);
2070     }
2071     set_control(_gvn.transform(ne_region));
2072   }
2073 }
2074 
2075 void Parse::do_acmp(BoolTest::mask btest, Node* left, Node* right) {
2076   ciKlass* left_type = nullptr;
2077   ciKlass* right_type = nullptr;
2078   ProfilePtrKind left_ptr = ProfileUnknownNull;
2079   ProfilePtrKind right_ptr = ProfileUnknownNull;
2080   bool left_inline_type = true;
2081   bool right_inline_type = true;
2082 
2083   // Leverage profiling at acmp
2084   if (UseACmpProfile) {
2085     method()->acmp_profiled_type(bci(), left_type, right_type, left_ptr, right_ptr, left_inline_type, right_inline_type);
2086     if (too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) {
2087       left_type = nullptr;
2088       right_type = nullptr;
2089       left_inline_type = true;
2090       right_inline_type = true;
2091     }
2092     if (too_many_traps_or_recompiles(Deoptimization::Reason_null_check)) {
2093       left_ptr = ProfileUnknownNull;
2094       right_ptr = ProfileUnknownNull;
2095     }
2096   }
2097 
2098   if (UseTypeSpeculation) {
2099     record_profile_for_speculation(left, left_type, left_ptr);
2100     record_profile_for_speculation(right, right_type, right_ptr);
2101   }
2102 
2103   if (!EnableValhalla) {
2104     Node* cmp = CmpP(left, right);
2105     cmp = optimize_cmp_with_klass(cmp);
2106     do_if(btest, cmp);
2107     return;
2108   }
2109 
2110   // Check for equality before potentially allocating
2111   if (left == right) {
2112     do_if(btest, makecon(TypeInt::CC_EQ));
2113     return;
2114   }
2115 
2116   // Allocate inline type operands and re-execute on deoptimization
2117   if (left->is_InlineType()) {
2118     if (_gvn.type(right)->is_zero_type() ||
2119         (right->is_InlineType() && _gvn.type(right->as_InlineType()->get_is_init())->is_zero_type())) {
2120       // Null checking a scalarized but nullable inline type. Check the IsInit
2121       // input instead of the oop input to avoid keeping buffer allocations alive.
2122       Node* cmp = CmpI(left->as_InlineType()->get_is_init(), intcon(0));
2123       do_if(btest, cmp);
2124       return;
2125     } else {
2126       PreserveReexecuteState preexecs(this);
2127       inc_sp(2);
2128       jvms()->set_should_reexecute(true);
2129       left = left->as_InlineType()->buffer(this)->get_oop();
2130     }
2131   }
2132   if (right->is_InlineType()) {
2133     PreserveReexecuteState preexecs(this);
2134     inc_sp(2);
2135     jvms()->set_should_reexecute(true);
2136     right = right->as_InlineType()->buffer(this)->get_oop();
2137   }
2138 
2139   // First, do a normal pointer comparison
2140   const TypeOopPtr* tleft = _gvn.type(left)->isa_oopptr();
2141   const TypeOopPtr* tright = _gvn.type(right)->isa_oopptr();
2142   Node* cmp = CmpP(left, right);
2143   cmp = optimize_cmp_with_klass(cmp);
2144   if (tleft == nullptr || !tleft->can_be_inline_type() ||
2145       tright == nullptr || !tright->can_be_inline_type()) {
2146     // This is sufficient, if one of the operands can't be an inline type
2147     do_if(btest, cmp);
2148     return;
2149   }
2150 
2151   // Don't add traps to unstable if branches because additional checks are required to
2152   // decide if the operands are equal/substitutable and we therefore shouldn't prune
2153   // branches for one if based on the profiling of the acmp branches.
2154   // Also, OptimizeUnstableIf would set an incorrect re-rexecution state because it
2155   // assumes that there is a 1-1 mapping between the if and the acmp branches and that
2156   // hitting a trap means that we will take the corresponding acmp branch on re-execution.
2157   const bool can_trap = true;
2158 
2159   Node* eq_region = nullptr;
2160   if (btest == BoolTest::eq) {
2161     do_if(btest, cmp, !can_trap, true);
2162     if (stopped()) {
2163       // Pointers are equal, operands must be equal
2164       return;
2165     }
2166   } else {
2167     assert(btest == BoolTest::ne, "only eq or ne");
2168     Node* is_not_equal = nullptr;
2169     eq_region = new RegionNode(3);
2170     {
2171       PreserveJVMState pjvms(this);
2172       // Pointers are not equal, but more checks are needed to determine if the operands are (not) substitutable
2173       do_if(btest, cmp, !can_trap, false, &is_not_equal);
2174       if (!stopped()) {
2175         eq_region->init_req(1, control());
2176       }
2177     }
2178     if (is_not_equal == nullptr || is_not_equal->is_top()) {
2179       record_for_igvn(eq_region);
2180       set_control(_gvn.transform(eq_region));
2181       return;
2182     }
2183     set_control(is_not_equal);
2184   }
2185 
2186   // Prefer speculative types if available
2187   if (!too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) {
2188     if (tleft->speculative_type() != nullptr) {
2189       left_type = tleft->speculative_type();
2190     }
2191     if (tright->speculative_type() != nullptr) {
2192       right_type = tright->speculative_type();
2193     }
2194   }
2195 
2196   if (speculative_ptr_kind(tleft) != ProfileMaybeNull && speculative_ptr_kind(tleft) != ProfileUnknownNull) {
2197     ProfilePtrKind speculative_left_ptr = speculative_ptr_kind(tleft);
2198     if (speculative_left_ptr == ProfileAlwaysNull && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_assert)) {
2199       left_ptr = speculative_left_ptr;
2200     } else if (speculative_left_ptr == ProfileNeverNull && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_check)) {
2201       left_ptr = speculative_left_ptr;
2202     }
2203   }
2204   if (speculative_ptr_kind(tright) != ProfileMaybeNull && speculative_ptr_kind(tright) != ProfileUnknownNull) {
2205     ProfilePtrKind speculative_right_ptr = speculative_ptr_kind(tright);
2206     if (speculative_right_ptr == ProfileAlwaysNull && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_assert)) {
2207       right_ptr = speculative_right_ptr;
2208     } else if (speculative_right_ptr == ProfileNeverNull && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_check)) {
2209       right_ptr = speculative_right_ptr;
2210     }
2211   }
2212 
2213   if (left_ptr == ProfileAlwaysNull) {
2214     // Comparison with null. Assert the input is indeed null and we're done.
2215     acmp_always_null_input(left, tleft, btest, eq_region);
2216     return;
2217   }
2218   if (right_ptr == ProfileAlwaysNull) {
2219     // Comparison with null. Assert the input is indeed null and we're done.
2220     acmp_always_null_input(right, tright, btest, eq_region);
2221     return;
2222   }
2223   if (left_type != nullptr && !left_type->is_inlinetype()) {
2224     // Comparison with an object of known type
2225     acmp_known_non_inline_type_input(left, tleft, left_ptr, left_type, btest, eq_region);
2226     return;
2227   }
2228   if (right_type != nullptr && !right_type->is_inlinetype()) {
2229     // Comparison with an object of known type
2230     acmp_known_non_inline_type_input(right, tright, right_ptr, right_type, btest, eq_region);
2231     return;
2232   }
2233   if (!left_inline_type) {
2234     // Comparison with an object known not to be an inline type
2235     acmp_unknown_non_inline_type_input(left, tleft, left_ptr, btest, eq_region);
2236     return;
2237   }
2238   if (!right_inline_type) {
2239     // Comparison with an object known not to be an inline type
2240     acmp_unknown_non_inline_type_input(right, tright, right_ptr, btest, eq_region);
2241     return;
2242   }
2243 
2244   // Pointers are not equal, check if first operand is non-null
2245   Node* ne_region = new RegionNode(6);
2246   Node* null_ctl;
2247   Node* not_null_right = acmp_null_check(right, tright, right_ptr, null_ctl);
2248   ne_region->init_req(1, null_ctl);
2249 
2250   // First operand is non-null, check if it is an inline type
2251   Node* is_value = inline_type_test(not_null_right);
2252   IfNode* is_value_iff = create_and_map_if(control(), is_value, PROB_FAIR, COUNT_UNKNOWN);
2253   Node* not_value = _gvn.transform(new IfFalseNode(is_value_iff));
2254   ne_region->init_req(2, not_value);
2255   set_control(_gvn.transform(new IfTrueNode(is_value_iff)));
2256 
2257   // The first operand is an inline type, check if the second operand is non-null
2258   Node* not_null_left = acmp_null_check(left, tleft, left_ptr, null_ctl);
2259   ne_region->init_req(3, null_ctl);
2260 
2261   // Check if both operands are of the same class.
2262   Node* kls_left = load_object_klass(not_null_left);
2263   Node* kls_right = load_object_klass(not_null_right);
2264   Node* kls_cmp = CmpP(kls_left, kls_right);
2265   Node* kls_bol = _gvn.transform(new BoolNode(kls_cmp, BoolTest::ne));
2266   IfNode* kls_iff = create_and_map_if(control(), kls_bol, PROB_FAIR, COUNT_UNKNOWN);
2267   Node* kls_ne = _gvn.transform(new IfTrueNode(kls_iff));
2268   set_control(_gvn.transform(new IfFalseNode(kls_iff)));
2269   ne_region->init_req(4, kls_ne);
2270 
2271   if (stopped()) {
2272     record_for_igvn(ne_region);
2273     set_control(_gvn.transform(ne_region));
2274     if (btest == BoolTest::ne) {
2275       {
2276         PreserveJVMState pjvms(this);
2277         int target_bci = iter().get_dest();
2278         merge(target_bci);
2279       }
2280       record_for_igvn(eq_region);
2281       set_control(_gvn.transform(eq_region));
2282     }
2283     return;
2284   }
2285 
2286   // Both operands are values types of the same class, we need to perform a
2287   // substitutability test. Delegate to ValueObjectMethods::isSubstitutable().
2288   Node* ne_io_phi = PhiNode::make(ne_region, i_o());
2289   Node* mem = reset_memory();
2290   Node* ne_mem_phi = PhiNode::make(ne_region, mem);
2291 
2292   Node* eq_io_phi = nullptr;
2293   Node* eq_mem_phi = nullptr;
2294   if (eq_region != nullptr) {
2295     eq_io_phi = PhiNode::make(eq_region, i_o());
2296     eq_mem_phi = PhiNode::make(eq_region, mem);
2297   }
2298 
2299   set_all_memory(mem);
2300 
2301   kill_dead_locals();
2302   ciMethod* subst_method = ciEnv::current()->ValueObjectMethods_klass()->find_method(ciSymbols::isSubstitutable_name(), ciSymbols::object_object_boolean_signature());
2303   CallStaticJavaNode *call = new CallStaticJavaNode(C, TypeFunc::make(subst_method), SharedRuntime::get_resolve_static_call_stub(), subst_method);
2304   call->set_override_symbolic_info(true);
2305   call->init_req(TypeFunc::Parms, not_null_left);
2306   call->init_req(TypeFunc::Parms+1, not_null_right);
2307   inc_sp(2);
2308   set_edges_for_java_call(call, false, false);
2309   Node* ret = set_results_for_java_call(call, false, true);
2310   dec_sp(2);
2311 
2312   // Test the return value of ValueObjectMethods::isSubstitutable()
2313   // This is the last check, do_if can emit traps now.
2314   Node* subst_cmp = _gvn.transform(new CmpINode(ret, intcon(1)));
2315   Node* ctl = C->top();
2316   if (btest == BoolTest::eq) {
2317     PreserveJVMState pjvms(this);
2318     do_if(btest, subst_cmp, can_trap);
2319     if (!stopped()) {
2320       ctl = control();
2321     }
2322   } else {
2323     assert(btest == BoolTest::ne, "only eq or ne");
2324     PreserveJVMState pjvms(this);
2325     do_if(btest, subst_cmp, can_trap, false, &ctl);
2326     if (!stopped()) {
2327       eq_region->init_req(2, control());
2328       eq_io_phi->init_req(2, i_o());
2329       eq_mem_phi->init_req(2, reset_memory());
2330     }
2331   }
2332   ne_region->init_req(5, ctl);
2333   ne_io_phi->init_req(5, i_o());
2334   ne_mem_phi->init_req(5, reset_memory());
2335 
2336   record_for_igvn(ne_region);
2337   set_control(_gvn.transform(ne_region));
2338   set_i_o(_gvn.transform(ne_io_phi));
2339   set_all_memory(_gvn.transform(ne_mem_phi));
2340 
2341   if (btest == BoolTest::ne) {
2342     {
2343       PreserveJVMState pjvms(this);
2344       int target_bci = iter().get_dest();
2345       merge(target_bci);
2346     }
2347 
2348     record_for_igvn(eq_region);
2349     set_control(_gvn.transform(eq_region));
2350     set_i_o(_gvn.transform(eq_io_phi));
2351     set_all_memory(_gvn.transform(eq_mem_phi));
2352   }
2353 }
2354 
2355 // Force unstable if traps to be taken randomly to trigger intermittent bugs such as incorrect debug information.
2356 // Add another if before the unstable if that checks a "random" condition at runtime (a simple shared counter) and
2357 // then either takes the trap or executes the original, unstable if.
2358 void Parse::stress_trap(IfNode* orig_iff, Node* counter, Node* incr_store) {
2359   // Search for an unstable if trap
2360   CallStaticJavaNode* trap = nullptr;
2361   assert(orig_iff->Opcode() == Op_If && orig_iff->outcnt() == 2, "malformed if");
2362   ProjNode* trap_proj = orig_iff->uncommon_trap_proj(trap, Deoptimization::Reason_unstable_if);
2363   if (trap == nullptr || !trap->jvms()->should_reexecute()) {
2364     // No suitable trap found. Remove unused counter load and increment.
2365     C->gvn_replace_by(incr_store, incr_store->in(MemNode::Memory));
2366     return;
2367   }
2368 
2369   // Remove trap from optimization list since we add another path to the trap.
2370   bool success = C->remove_unstable_if_trap(trap, true);
2371   assert(success, "Trap already modified");
2372 
2373   // Add a check before the original if that will trap with a certain frequency and execute the original if otherwise
2374   int freq_log = (C->random() % 31) + 1; // Random logarithmic frequency in [1, 31]

2407 }
2408 
2409 void Parse::maybe_add_predicate_after_if(Block* path) {
2410   if (path->is_SEL_head() && path->preds_parsed() == 0) {
2411     // Add predicates at bci of if dominating the loop so traps can be
2412     // recorded on the if's profile data
2413     int bc_depth = repush_if_args();
2414     add_parse_predicates();
2415     dec_sp(bc_depth);
2416     path->set_has_predicates();
2417   }
2418 }
2419 
2420 
2421 //----------------------------adjust_map_after_if------------------------------
2422 // Adjust the JVM state to reflect the result of taking this path.
2423 // Basically, it means inspecting the CmpNode controlling this
2424 // branch, seeing how it constrains a tested value, and then
2425 // deciding if it's worth our while to encode this constraint
2426 // as graph nodes in the current abstract interpretation map.
2427 void Parse::adjust_map_after_if(BoolTest::mask btest, Node* c, float prob, Block* path, bool can_trap) {
2428   if (!c->is_Cmp()) {
2429     maybe_add_predicate_after_if(path);
2430     return;
2431   }
2432 
2433   if (stopped() || btest == BoolTest::illegal) {
2434     return;                             // nothing to do
2435   }
2436 
2437   bool is_fallthrough = (path == successor_for_bci(iter().next_bci()));
2438 
2439   if (can_trap && path_is_suitable_for_uncommon_trap(prob)) {
2440     repush_if_args();
2441     Node* call = uncommon_trap(Deoptimization::Reason_unstable_if,
2442                   Deoptimization::Action_reinterpret,
2443                   nullptr,
2444                   (is_fallthrough ? "taken always" : "taken never"));
2445 
2446     if (call != nullptr) {
2447       C->record_unstable_if_trap(new UnstableIfTrap(call->as_CallStaticJava(), path));
2448     }
2449     return;
2450   }
2451 
2452   Node* val = c->in(1);
2453   Node* con = c->in(2);
2454   const Type* tcon = _gvn.type(con);
2455   const Type* tval = _gvn.type(val);
2456   bool have_con = tcon->singleton();
2457   if (tval->singleton()) {
2458     if (!have_con) {
2459       // Swap, so constant is in con.

2516     if (obj != nullptr && (con_type->isa_instptr() || con_type->isa_aryptr())) {
2517        // Found:
2518        //   Bool(CmpP(LoadKlass(obj._klass), ConP(Foo.klass)), [eq])
2519        // or the narrowOop equivalent.
2520        const Type* obj_type = _gvn.type(obj);
2521        const TypeOopPtr* tboth = obj_type->join_speculative(con_type)->isa_oopptr();
2522        if (tboth != nullptr && tboth->klass_is_exact() && tboth != obj_type &&
2523            tboth->higher_equal(obj_type)) {
2524           // obj has to be of the exact type Foo if the CmpP succeeds.
2525           int obj_in_map = map()->find_edge(obj);
2526           JVMState* jvms = this->jvms();
2527           if (obj_in_map >= 0 &&
2528               (jvms->is_loc(obj_in_map) || jvms->is_stk(obj_in_map))) {
2529             TypeNode* ccast = new CheckCastPPNode(control(), obj, tboth);
2530             const Type* tcc = ccast->as_Type()->type();
2531             assert(tcc != obj_type && tcc->higher_equal(obj_type), "must improve");
2532             // Delay transform() call to allow recovery of pre-cast value
2533             // at the control merge.
2534             _gvn.set_type_bottom(ccast);
2535             record_for_igvn(ccast);
2536             if (tboth->is_inlinetypeptr()) {
2537               ccast = InlineTypeNode::make_from_oop(this, ccast, tboth->exact_klass(true)->as_inline_klass());
2538             }
2539             // Here's the payoff.
2540             replace_in_map(obj, ccast);
2541           }
2542        }
2543     }
2544   }
2545 
2546   int val_in_map = map()->find_edge(val);
2547   if (val_in_map < 0)  return;          // replace_in_map would be useless
2548   {
2549     JVMState* jvms = this->jvms();
2550     if (!(jvms->is_loc(val_in_map) ||
2551           jvms->is_stk(val_in_map)))
2552       return;                           // again, it would be useless
2553   }
2554 
2555   // Check for a comparison to a constant, and "know" that the compared
2556   // value is constrained on this path.
2557   assert(tcon->singleton(), "");
2558   ConstraintCastNode* ccast = nullptr;

2623   if (c->Opcode() == Op_CmpP &&
2624       (c->in(1)->Opcode() == Op_LoadKlass || c->in(1)->Opcode() == Op_DecodeNKlass) &&
2625       c->in(2)->is_Con()) {
2626     Node* load_klass = nullptr;
2627     Node* decode = nullptr;
2628     if (c->in(1)->Opcode() == Op_DecodeNKlass) {
2629       decode = c->in(1);
2630       load_klass = c->in(1)->in(1);
2631     } else {
2632       load_klass = c->in(1);
2633     }
2634     if (load_klass->in(2)->is_AddP()) {
2635       Node* addp = load_klass->in(2);
2636       Node* obj = addp->in(AddPNode::Address);
2637       const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr();
2638       if (obj_type->speculative_type_not_null() != nullptr) {
2639         ciKlass* k = obj_type->speculative_type();
2640         inc_sp(2);
2641         obj = maybe_cast_profiled_obj(obj, k);
2642         dec_sp(2);
2643         if (obj->is_InlineType()) {
2644           assert(obj->as_InlineType()->is_allocated(&_gvn), "must be allocated");
2645           obj = obj->as_InlineType()->get_oop();
2646         }
2647         // Make the CmpP use the casted obj
2648         addp = basic_plus_adr(obj, addp->in(AddPNode::Offset));
2649         load_klass = load_klass->clone();
2650         load_klass->set_req(2, addp);
2651         load_klass = _gvn.transform(load_klass);
2652         if (decode != nullptr) {
2653           decode = decode->clone();
2654           decode->set_req(1, load_klass);
2655           load_klass = _gvn.transform(decode);
2656         }
2657         c = c->clone();
2658         c->set_req(1, load_klass);
2659         c = _gvn.transform(c);
2660       }
2661     }
2662   }
2663   return c;
2664 }
2665 
2666 //------------------------------do_one_bytecode--------------------------------

3366 
3367   case Bytecodes::_i2d:
3368     a = pop();
3369     b = _gvn.transform( new ConvI2DNode(a));
3370     push_pair(b);
3371     break;
3372 
3373   case Bytecodes::_iinc:        // Increment local
3374     i = iter().get_index();     // Get local index
3375     set_local( i, _gvn.transform( new AddINode( _gvn.intcon(iter().get_iinc_con()), local(i) ) ) );
3376     break;
3377 
3378   // Exit points of synchronized methods must have an unlock node
3379   case Bytecodes::_return:
3380     return_current(nullptr);
3381     break;
3382 
3383   case Bytecodes::_ireturn:
3384   case Bytecodes::_areturn:
3385   case Bytecodes::_freturn:
3386     return_current(cast_to_non_larval(pop()));
3387     break;
3388   case Bytecodes::_lreturn:


3389   case Bytecodes::_dreturn:
3390     return_current(pop_pair());
3391     break;
3392 
3393   case Bytecodes::_athrow:
3394     // null exception oop throws null pointer exception
3395     null_check(peek());
3396     if (stopped())  return;
3397     // Hook the thrown exception directly to subsequent handlers.
3398     if (BailoutToInterpreterForThrows) {
3399       // Keep method interpreted from now on.
3400       uncommon_trap(Deoptimization::Reason_unhandled,
3401                     Deoptimization::Action_make_not_compilable);
3402       return;
3403     }
3404     if (env()->jvmti_can_post_on_exceptions()) {
3405       // check if we must post exception events, take uncommon trap if so (with must_throw = false)
3406       uncommon_trap_if_should_post_on_exceptions(Deoptimization::Reason_unhandled, false);
3407     }
3408     // Here if either can_post_on_exceptions or should_post_on_exceptions is false

3421 
3422     // See if we can get some profile data and hand it off to the next block
3423     Block *target_block = block()->successor_for_bci(target_bci);
3424     if (target_block->pred_count() != 1)  break;
3425     ciMethodData* methodData = method()->method_data();
3426     if (!methodData->is_mature())  break;
3427     ciProfileData* data = methodData->bci_to_data(bci());
3428     assert(data != nullptr && data->is_JumpData(), "need JumpData for taken branch");
3429     int taken = ((ciJumpData*)data)->taken();
3430     taken = method()->scale_count(taken);
3431     target_block->set_count(taken);
3432     break;
3433   }
3434 
3435   case Bytecodes::_ifnull:    btest = BoolTest::eq; goto handle_if_null;
3436   case Bytecodes::_ifnonnull: btest = BoolTest::ne; goto handle_if_null;
3437   handle_if_null:
3438     // If this is a backwards branch in the bytecodes, add Safepoint
3439     maybe_add_safepoint(iter().get_dest());
3440     a = null();
3441     b = cast_to_non_larval(pop());
3442     if (b->is_InlineType()) {
3443       // Null checking a scalarized but nullable inline type. Check the IsInit
3444       // input instead of the oop input to avoid keeping buffer allocations alive
3445       c = _gvn.transform(new CmpINode(b->as_InlineType()->get_is_init(), zerocon(T_INT)));
3446     } else {
3447       if (!_gvn.type(b)->speculative_maybe_null() &&
3448           !too_many_traps(Deoptimization::Reason_speculate_null_check)) {
3449         inc_sp(1);
3450         Node* null_ctl = top();
3451         b = null_check_oop(b, &null_ctl, true, true, true);
3452         assert(null_ctl->is_top(), "no null control here");
3453         dec_sp(1);
3454       } else if (_gvn.type(b)->speculative_always_null() &&
3455                  !too_many_traps(Deoptimization::Reason_speculate_null_assert)) {
3456         inc_sp(1);
3457         b = null_assert(b);
3458         dec_sp(1);
3459       }
3460       c = _gvn.transform( new CmpPNode(b, a) );
3461     }
3462     do_ifnull(btest, c);
3463     break;
3464 
3465   case Bytecodes::_if_acmpeq: btest = BoolTest::eq; goto handle_if_acmp;
3466   case Bytecodes::_if_acmpne: btest = BoolTest::ne; goto handle_if_acmp;
3467   handle_if_acmp:
3468     // If this is a backwards branch in the bytecodes, add Safepoint
3469     maybe_add_safepoint(iter().get_dest());
3470     a = cast_to_non_larval(pop());
3471     b = cast_to_non_larval(pop());
3472     do_acmp(btest, b, a);


3473     break;
3474 
3475   case Bytecodes::_ifeq: btest = BoolTest::eq; goto handle_ifxx;
3476   case Bytecodes::_ifne: btest = BoolTest::ne; goto handle_ifxx;
3477   case Bytecodes::_iflt: btest = BoolTest::lt; goto handle_ifxx;
3478   case Bytecodes::_ifle: btest = BoolTest::le; goto handle_ifxx;
3479   case Bytecodes::_ifgt: btest = BoolTest::gt; goto handle_ifxx;
3480   case Bytecodes::_ifge: btest = BoolTest::ge; goto handle_ifxx;
3481   handle_ifxx:
3482     // If this is a backwards branch in the bytecodes, add Safepoint
3483     maybe_add_safepoint(iter().get_dest());
3484     a = _gvn.intcon(0);
3485     b = pop();
3486     c = _gvn.transform( new CmpINode(b, a) );
3487     do_if(btest, c);
3488     break;
3489 
3490   case Bytecodes::_if_icmpeq: btest = BoolTest::eq; goto handle_if_icmp;
3491   case Bytecodes::_if_icmpne: btest = BoolTest::ne; goto handle_if_icmp;
3492   case Bytecodes::_if_icmplt: btest = BoolTest::lt; goto handle_if_icmp;

3507     break;
3508 
3509   case Bytecodes::_lookupswitch:
3510     do_lookupswitch();
3511     break;
3512 
3513   case Bytecodes::_invokestatic:
3514   case Bytecodes::_invokedynamic:
3515   case Bytecodes::_invokespecial:
3516   case Bytecodes::_invokevirtual:
3517   case Bytecodes::_invokeinterface:
3518     do_call();
3519     break;
3520   case Bytecodes::_checkcast:
3521     do_checkcast();
3522     break;
3523   case Bytecodes::_instanceof:
3524     do_instanceof();
3525     break;
3526   case Bytecodes::_anewarray:
3527     do_newarray();
3528     break;
3529   case Bytecodes::_newarray:
3530     do_newarray((BasicType)iter().get_index());
3531     break;
3532   case Bytecodes::_multianewarray:
3533     do_multianewarray();
3534     break;
3535   case Bytecodes::_new:
3536     do_new();
3537     break;
3538 
3539   case Bytecodes::_jsr:
3540   case Bytecodes::_jsr_w:
3541     do_jsr();
3542     break;
3543 
3544   case Bytecodes::_ret:
3545     do_ret();
3546     break;
3547 

3557   case Bytecodes::_breakpoint:
3558     // Breakpoint set concurrently to compile
3559     // %%% use an uncommon trap?
3560     C->record_failure("breakpoint in method");
3561     return;
3562 
3563   default:
3564 #ifndef PRODUCT
3565     map()->dump(99);
3566 #endif
3567     tty->print("\nUnhandled bytecode %s\n", Bytecodes::name(bc()) );
3568     ShouldNotReachHere();
3569   }
3570 
3571 #ifndef PRODUCT
3572   if (failing()) { return; }
3573   constexpr int perBytecode = 6;
3574   if (C->should_print_igv(perBytecode)) {
3575     IdealGraphPrinter* printer = C->igv_printer();
3576     char buffer[256];
3577     jio_snprintf(buffer, sizeof(buffer), "Bytecode %d: %s, map: %d", bci(), Bytecodes::name(bc()), map() == nullptr ? -1 : map()->_idx);
3578     bool old = printer->traverse_outs();
3579     printer->set_traverse_outs(true);
3580     printer->print_graph(buffer);
3581     printer->set_traverse_outs(old);
3582   }
3583 #endif
3584 }
< prev index next >