< prev index next >

src/hotspot/share/opto/graphKit.cpp

Print this page

   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"


  26 #include "ci/ciUtilities.hpp"
  27 #include "classfile/javaClasses.hpp"
  28 #include "ci/ciObjArray.hpp"
  29 #include "asm/register.hpp"
  30 #include "compiler/compileLog.hpp"
  31 #include "gc/shared/barrierSet.hpp"
  32 #include "gc/shared/c2/barrierSetC2.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "opto/addnode.hpp"
  36 #include "opto/castnode.hpp"
  37 #include "opto/convertnode.hpp"
  38 #include "opto/graphKit.hpp"
  39 #include "opto/idealKit.hpp"

  40 #include "opto/intrinsicnode.hpp"
  41 #include "opto/locknode.hpp"
  42 #include "opto/machnode.hpp"

  43 #include "opto/opaquenode.hpp"
  44 #include "opto/parse.hpp"
  45 #include "opto/rootnode.hpp"
  46 #include "opto/runtime.hpp"
  47 #include "opto/subtypenode.hpp"
  48 #include "runtime/deoptimization.hpp"
  49 #include "runtime/sharedRuntime.hpp"
  50 #include "utilities/bitMap.inline.hpp"
  51 #include "utilities/powerOfTwo.hpp"
  52 #include "utilities/growableArray.hpp"
  53 
  54 //----------------------------GraphKit-----------------------------------------
  55 // Main utility constructor.
  56 GraphKit::GraphKit(JVMState* jvms)
  57   : Phase(Phase::Parser),
  58     _env(C->env()),
  59     _gvn(*C->initial_gvn()),
  60     _barrier_set(BarrierSet::barrier_set()->barrier_set_c2())
  61 {

  62   _exceptions = jvms->map()->next_exception();
  63   if (_exceptions != nullptr)  jvms->map()->set_next_exception(nullptr);
  64   set_jvms(jvms);







  65 }
  66 
  67 // Private constructor for parser.
  68 GraphKit::GraphKit()
  69   : Phase(Phase::Parser),
  70     _env(C->env()),
  71     _gvn(*C->initial_gvn()),
  72     _barrier_set(BarrierSet::barrier_set()->barrier_set_c2())
  73 {
  74   _exceptions = nullptr;
  75   set_map(nullptr);
  76   debug_only(_sp = -99);
  77   debug_only(set_bci(-99));
  78 }
  79 
  80 
  81 
  82 //---------------------------clean_stack---------------------------------------
  83 // Clear away rubbish from the stack area of the JVM state.
  84 // This destroys any arguments that may be waiting on the stack.

 840         if (PrintMiscellaneous && (Verbose || WizardMode)) {
 841           tty->print_cr("Zombie local %d: ", local);
 842           jvms->dump();
 843         }
 844         return false;
 845       }
 846     }
 847   }
 848   return true;
 849 }
 850 
 851 #endif //ASSERT
 852 
 853 // Helper function for enforcing certain bytecodes to reexecute if deoptimization happens.
 854 static bool should_reexecute_implied_by_bytecode(JVMState *jvms, bool is_anewarray) {
 855   ciMethod* cur_method = jvms->method();
 856   int       cur_bci   = jvms->bci();
 857   if (cur_method != nullptr && cur_bci != InvocationEntryBci) {
 858     Bytecodes::Code code = cur_method->java_code_at_bci(cur_bci);
 859     return Interpreter::bytecode_should_reexecute(code) ||
 860            (is_anewarray && code == Bytecodes::_multianewarray);
 861     // Reexecute _multianewarray bytecode which was replaced with
 862     // sequence of [a]newarray. See Parse::do_multianewarray().
 863     //
 864     // Note: interpreter should not have it set since this optimization
 865     // is limited by dimensions and guarded by flag so in some cases
 866     // multianewarray() runtime calls will be generated and
 867     // the bytecode should not be reexecutes (stack will not be reset).
 868   } else {
 869     return false;
 870   }
 871 }
 872 
 873 // Helper function for adding JVMState and debug information to node
 874 void GraphKit::add_safepoint_edges(SafePointNode* call, bool must_throw) {
 875   // Add the safepoint edges to the call (or other safepoint).
 876 
 877   // Make sure dead locals are set to top.  This
 878   // should help register allocation time and cut down on the size
 879   // of the deoptimization information.
 880   assert(dead_locals_are_killed(), "garbage in debug info before safepoint");

1182   Node* conv = _gvn.transform( new ConvI2LNode(offset));
1183   Node* mask = _gvn.transform(ConLNode::make((julong) max_juint));
1184   return _gvn.transform( new AndLNode(conv, mask) );
1185 }
1186 
1187 Node* GraphKit::ConvL2I(Node* offset) {
1188   // short-circuit a common case
1189   jlong offset_con = find_long_con(offset, (jlong)Type::OffsetBot);
1190   if (offset_con != (jlong)Type::OffsetBot) {
1191     return intcon((int) offset_con);
1192   }
1193   return _gvn.transform( new ConvL2INode(offset));
1194 }
1195 
1196 //-------------------------load_object_klass-----------------------------------
1197 Node* GraphKit::load_object_klass(Node* obj) {
1198   // Special-case a fresh allocation to avoid building nodes:
1199   Node* akls = AllocateNode::Ideal_klass(obj, &_gvn);
1200   if (akls != nullptr)  return akls;
1201   Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
1202   return _gvn.transform(LoadKlassNode::make(_gvn, nullptr, immutable_memory(), k_adr, TypeInstPtr::KLASS));
1203 }
1204 
1205 //-------------------------load_array_length-----------------------------------
1206 Node* GraphKit::load_array_length(Node* array) {
1207   // Special-case a fresh allocation to avoid building nodes:
1208   AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(array);
1209   Node *alen;
1210   if (alloc == nullptr) {
1211     Node *r_adr = basic_plus_adr(array, arrayOopDesc::length_offset_in_bytes());
1212     alen = _gvn.transform( new LoadRangeNode(0, immutable_memory(), r_adr, TypeInt::POS));
1213   } else {
1214     alen = array_ideal_length(alloc, _gvn.type(array)->is_oopptr(), false);
1215   }
1216   return alen;
1217 }
1218 
1219 Node* GraphKit::array_ideal_length(AllocateArrayNode* alloc,
1220                                    const TypeOopPtr* oop_type,
1221                                    bool replace_length_in_map) {
1222   Node* length = alloc->Ideal_length();

1231         replace_in_map(length, ccast);
1232       }
1233       return ccast;
1234     }
1235   }
1236   return length;
1237 }
1238 
1239 //------------------------------do_null_check----------------------------------
1240 // Helper function to do a null pointer check.  Returned value is
1241 // the incoming address with null casted away.  You are allowed to use the
1242 // not-null value only if you are control dependent on the test.
1243 #ifndef PRODUCT
1244 extern uint explicit_null_checks_inserted,
1245             explicit_null_checks_elided;
1246 #endif
1247 Node* GraphKit::null_check_common(Node* value, BasicType type,
1248                                   // optional arguments for variations:
1249                                   bool assert_null,
1250                                   Node* *null_control,
1251                                   bool speculative) {

1252   assert(!assert_null || null_control == nullptr, "not both at once");
1253   if (stopped())  return top();
1254   NOT_PRODUCT(explicit_null_checks_inserted++);
1255 






















1256   // Construct null check
1257   Node *chk = nullptr;
1258   switch(type) {
1259     case T_LONG   : chk = new CmpLNode(value, _gvn.zerocon(T_LONG)); break;
1260     case T_INT    : chk = new CmpINode(value, _gvn.intcon(0)); break;
1261     case T_ARRAY  : // fall through
1262       type = T_OBJECT;  // simplify further tests
1263     case T_OBJECT : {
1264       const Type *t = _gvn.type( value );
1265 
1266       const TypeOopPtr* tp = t->isa_oopptr();
1267       if (tp != nullptr && !tp->is_loaded()
1268           // Only for do_null_check, not any of its siblings:
1269           && !assert_null && null_control == nullptr) {
1270         // Usually, any field access or invocation on an unloaded oop type
1271         // will simply fail to link, since the statically linked class is
1272         // likely also to be unloaded.  However, in -Xcomp mode, sometimes
1273         // the static class is loaded but the sharper oop type is not.
1274         // Rather than checking for this obscure case in lots of places,
1275         // we simply observe that a null check on an unloaded class

1339         }
1340         Node *oldcontrol = control();
1341         set_control(cfg);
1342         Node *res = cast_not_null(value);
1343         set_control(oldcontrol);
1344         NOT_PRODUCT(explicit_null_checks_elided++);
1345         return res;
1346       }
1347       cfg = IfNode::up_one_dom(cfg, /*linear_only=*/ true);
1348       if (cfg == nullptr)  break;  // Quit at region nodes
1349       depth++;
1350     }
1351   }
1352 
1353   //-----------
1354   // Branch to failure if null
1355   float ok_prob = PROB_MAX;  // a priori estimate:  nulls never happen
1356   Deoptimization::DeoptReason reason;
1357   if (assert_null) {
1358     reason = Deoptimization::reason_null_assert(speculative);
1359   } else if (type == T_OBJECT) {
1360     reason = Deoptimization::reason_null_check(speculative);
1361   } else {
1362     reason = Deoptimization::Reason_div0_check;
1363   }
1364   // %%% Since Reason_unhandled is not recorded on a per-bytecode basis,
1365   // ciMethodData::has_trap_at will return a conservative -1 if any
1366   // must-be-null assertion has failed.  This could cause performance
1367   // problems for a method after its first do_null_assert failure.
1368   // Consider using 'Reason_class_check' instead?
1369 
1370   // To cause an implicit null check, we set the not-null probability
1371   // to the maximum (PROB_MAX).  For an explicit check the probability
1372   // is set to a smaller value.
1373   if (null_control != nullptr || too_many_traps(reason)) {
1374     // probability is less likely
1375     ok_prob =  PROB_LIKELY_MAG(3);
1376   } else if (!assert_null &&
1377              (ImplicitNullCheckThreshold > 0) &&
1378              method() != nullptr &&
1379              (method()->method_data()->trap_count(reason)

1413   }
1414 
1415   if (assert_null) {
1416     // Cast obj to null on this path.
1417     replace_in_map(value, zerocon(type));
1418     return zerocon(type);
1419   }
1420 
1421   // Cast obj to not-null on this path, if there is no null_control.
1422   // (If there is a null_control, a non-null value may come back to haunt us.)
1423   if (type == T_OBJECT) {
1424     Node* cast = cast_not_null(value, false);
1425     if (null_control == nullptr || (*null_control) == top())
1426       replace_in_map(value, cast);
1427     value = cast;
1428   }
1429 
1430   return value;
1431 }
1432 
1433 
1434 //------------------------------cast_not_null----------------------------------
1435 // Cast obj to not-null on this path
1436 Node* GraphKit::cast_not_null(Node* obj, bool do_replace_in_map) {









1437   const Type *t = _gvn.type(obj);
1438   const Type *t_not_null = t->join_speculative(TypePtr::NOTNULL);
1439   // Object is already not-null?
1440   if( t == t_not_null ) return obj;
1441 
1442   Node *cast = new CastPPNode(obj,t_not_null);
1443   cast->init_req(0, control());
1444   cast = _gvn.transform( cast );
1445 
1446   // Scan for instances of 'obj' in the current JVM mapping.
1447   // These instances are known to be not-null after the test.
1448   if (do_replace_in_map)
1449     replace_in_map(obj, cast);
1450 
1451   return cast;                  // Return casted value
1452 }
1453 
1454 // Sometimes in intrinsics, we implicitly know an object is not null
1455 // (there's no actual null check) so we can cast it to not null. In
1456 // the course of optimizations, the input to the cast can become null.

1543 // These are layered on top of the factory methods in LoadNode and StoreNode,
1544 // and integrate with the parser's memory state and _gvn engine.
1545 //
1546 
1547 // factory methods in "int adr_idx"
1548 Node* GraphKit::make_load(Node* ctl, Node* adr, const Type* t, BasicType bt,
1549                           int adr_idx,
1550                           MemNode::MemOrd mo,
1551                           LoadNode::ControlDependency control_dependency,
1552                           bool require_atomic_access,
1553                           bool unaligned,
1554                           bool mismatched,
1555                           bool unsafe,
1556                           uint8_t barrier_data) {
1557   assert(adr_idx != Compile::AliasIdxTop, "use other make_load factory" );
1558   const TypePtr* adr_type = nullptr; // debug-mode-only argument
1559   debug_only(adr_type = C->get_adr_type(adr_idx));
1560   Node* mem = memory(adr_idx);
1561   Node* ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt, mo, control_dependency, require_atomic_access, unaligned, mismatched, unsafe, barrier_data);
1562   ld = _gvn.transform(ld);

1563   if (((bt == T_OBJECT) && C->do_escape_analysis()) || C->eliminate_boxing()) {
1564     // Improve graph before escape analysis and boxing elimination.
1565     record_for_igvn(ld);
1566   }
1567   return ld;
1568 }
1569 
1570 Node* GraphKit::store_to_memory(Node* ctl, Node* adr, Node *val, BasicType bt,
1571                                 int adr_idx,
1572                                 MemNode::MemOrd mo,
1573                                 bool require_atomic_access,
1574                                 bool unaligned,
1575                                 bool mismatched,
1576                                 bool unsafe,
1577                                 int barrier_data) {
1578   assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
1579   const TypePtr* adr_type = nullptr;
1580   debug_only(adr_type = C->get_adr_type(adr_idx));
1581   Node *mem = memory(adr_idx);
1582   Node* st = StoreNode::make(_gvn, ctl, mem, adr, adr_type, val, bt, mo, require_atomic_access);

1589   if (unsafe) {
1590     st->as_Store()->set_unsafe_access();
1591   }
1592   st->as_Store()->set_barrier_data(barrier_data);
1593   st = _gvn.transform(st);
1594   set_memory(st, adr_idx);
1595   // Back-to-back stores can only remove intermediate store with DU info
1596   // so push on worklist for optimizer.
1597   if (mem->req() > MemNode::Address && adr == mem->in(MemNode::Address))
1598     record_for_igvn(st);
1599 
1600   return st;
1601 }
1602 
1603 Node* GraphKit::access_store_at(Node* obj,
1604                                 Node* adr,
1605                                 const TypePtr* adr_type,
1606                                 Node* val,
1607                                 const Type* val_type,
1608                                 BasicType bt,
1609                                 DecoratorSet decorators) {

1610   // Transformation of a value which could be null pointer (CastPP #null)
1611   // could be delayed during Parse (for example, in adjust_map_after_if()).
1612   // Execute transformation here to avoid barrier generation in such case.
1613   if (_gvn.type(val) == TypePtr::NULL_PTR) {
1614     val = _gvn.makecon(TypePtr::NULL_PTR);
1615   }
1616 
1617   if (stopped()) {
1618     return top(); // Dead path ?
1619   }
1620 
1621   assert(val != nullptr, "not dead path");







1622 
1623   C2AccessValuePtr addr(adr, adr_type);
1624   C2AccessValue value(val, val_type);
1625   C2ParseAccess access(this, decorators | C2_WRITE_ACCESS, bt, obj, addr);
1626   if (access.is_raw()) {
1627     return _barrier_set->BarrierSetC2::store_at(access, value);
1628   } else {
1629     return _barrier_set->store_at(access, value);
1630   }
1631 }
1632 
1633 Node* GraphKit::access_load_at(Node* obj,   // containing obj
1634                                Node* adr,   // actual address to store val at
1635                                const TypePtr* adr_type,
1636                                const Type* val_type,
1637                                BasicType bt,
1638                                DecoratorSet decorators) {

1639   if (stopped()) {
1640     return top(); // Dead path ?
1641   }
1642 
1643   C2AccessValuePtr addr(adr, adr_type);
1644   C2ParseAccess access(this, decorators | C2_READ_ACCESS, bt, obj, addr);
1645   if (access.is_raw()) {
1646     return _barrier_set->BarrierSetC2::load_at(access, val_type);
1647   } else {
1648     return _barrier_set->load_at(access, val_type);
1649   }
1650 }
1651 
1652 Node* GraphKit::access_load(Node* adr,   // actual address to load val at
1653                             const Type* val_type,
1654                             BasicType bt,
1655                             DecoratorSet decorators) {
1656   if (stopped()) {
1657     return top(); // Dead path ?
1658   }
1659 
1660   C2AccessValuePtr addr(adr, adr->bottom_type()->is_ptr());
1661   C2ParseAccess access(this, decorators | C2_READ_ACCESS, bt, nullptr, addr);
1662   if (access.is_raw()) {
1663     return _barrier_set->BarrierSetC2::load_at(access, val_type);
1664   } else {

1729                                      Node* new_val,
1730                                      const Type* value_type,
1731                                      BasicType bt,
1732                                      DecoratorSet decorators) {
1733   C2AccessValuePtr addr(adr, adr_type);
1734   C2AtomicParseAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, bt, obj, addr, alias_idx);
1735   if (access.is_raw()) {
1736     return _barrier_set->BarrierSetC2::atomic_add_at(access, new_val, value_type);
1737   } else {
1738     return _barrier_set->atomic_add_at(access, new_val, value_type);
1739   }
1740 }
1741 
1742 void GraphKit::access_clone(Node* src, Node* dst, Node* size, bool is_array) {
1743   return _barrier_set->clone(this, src, dst, size, is_array);
1744 }
1745 
1746 //-------------------------array_element_address-------------------------
1747 Node* GraphKit::array_element_address(Node* ary, Node* idx, BasicType elembt,
1748                                       const TypeInt* sizetype, Node* ctrl) {
1749   uint shift  = exact_log2(type2aelembytes(elembt));

1750   uint header = arrayOopDesc::base_offset_in_bytes(elembt);
1751 
1752   // short-circuit a common case (saves lots of confusing waste motion)
1753   jint idx_con = find_int_con(idx, -1);
1754   if (idx_con >= 0) {
1755     intptr_t offset = header + ((intptr_t)idx_con << shift);
1756     return basic_plus_adr(ary, offset);
1757   }
1758 
1759   // must be correct type for alignment purposes
1760   Node* base  = basic_plus_adr(ary, header);
1761   idx = Compile::conv_I2X_index(&_gvn, idx, sizetype, ctrl);
1762   Node* scale = _gvn.transform( new LShiftXNode(idx, intcon(shift)) );
1763   return basic_plus_adr(ary, base, scale);
1764 }
1765 
1766 //-------------------------load_array_element-------------------------
1767 Node* GraphKit::load_array_element(Node* ary, Node* idx, const TypeAryPtr* arytype, bool set_ctrl) {
1768   const Type* elemtype = arytype->elem();
1769   BasicType elembt = elemtype->array_element_basic_type();
1770   Node* adr = array_element_address(ary, idx, elembt, arytype->size());
1771   if (elembt == T_NARROWOOP) {
1772     elembt = T_OBJECT; // To satisfy switch in LoadNode::make()
1773   }
1774   Node* ld = access_load_at(ary, adr, arytype, elemtype, elembt,
1775                             IN_HEAP | IS_ARRAY | (set_ctrl ? C2_CONTROL_DEPENDENT_LOAD : 0));
1776   return ld;
1777 }
1778 
1779 //-------------------------set_arguments_for_java_call-------------------------
1780 // Arguments (pre-popped from the stack) are taken from the JVMS.
1781 void GraphKit::set_arguments_for_java_call(CallJavaNode* call) {
1782   // Add the call arguments:
1783   uint nargs = call->method()->arg_size();
1784   for (uint i = 0; i < nargs; i++) {
1785     Node* arg = argument(i);
1786     call->init_req(i + TypeFunc::Parms, arg);







































1787   }
1788 }
1789 
1790 //---------------------------set_edges_for_java_call---------------------------
1791 // Connect a newly created call into the current JVMS.
1792 // A return value node (if any) is returned from set_edges_for_java_call.
1793 void GraphKit::set_edges_for_java_call(CallJavaNode* call, bool must_throw, bool separate_io_proj) {
1794 
1795   // Add the predefined inputs:
1796   call->init_req( TypeFunc::Control, control() );
1797   call->init_req( TypeFunc::I_O    , i_o() );
1798   call->init_req( TypeFunc::Memory , reset_memory() );
1799   call->init_req( TypeFunc::FramePtr, frameptr() );
1800   call->init_req( TypeFunc::ReturnAdr, top() );
1801 
1802   add_safepoint_edges(call, must_throw);
1803 
1804   Node* xcall = _gvn.transform(call);
1805 
1806   if (xcall == top()) {
1807     set_control(top());
1808     return;
1809   }
1810   assert(xcall == call, "call identity is stable");
1811 
1812   // Re-use the current map to produce the result.
1813 
1814   set_control(_gvn.transform(new ProjNode(call, TypeFunc::Control)));
1815   set_i_o(    _gvn.transform(new ProjNode(call, TypeFunc::I_O    , separate_io_proj)));
1816   set_all_memory_call(xcall, separate_io_proj);
1817 
1818   //return xcall;   // no need, caller already has it
1819 }
1820 
1821 Node* GraphKit::set_results_for_java_call(CallJavaNode* call, bool separate_io_proj, bool deoptimize) {
1822   if (stopped())  return top();  // maybe the call folded up?
1823 
1824   // Capture the return value, if any.
1825   Node* ret;
1826   if (call->method() == nullptr ||
1827       call->method()->return_type()->basic_type() == T_VOID)
1828         ret = top();
1829   else  ret = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
1830 
1831   // Note:  Since any out-of-line call can produce an exception,
1832   // we always insert an I_O projection from the call into the result.
1833 
1834   make_slow_call_ex(call, env()->Throwable_klass(), separate_io_proj, deoptimize);
1835 
1836   if (separate_io_proj) {
1837     // The caller requested separate projections be used by the fall
1838     // through and exceptional paths, so replace the projections for
1839     // the fall through path.
1840     set_i_o(_gvn.transform( new ProjNode(call, TypeFunc::I_O) ));
1841     set_all_memory(_gvn.transform( new ProjNode(call, TypeFunc::Memory) ));
1842   }























1843   return ret;
1844 }
1845 
1846 //--------------------set_predefined_input_for_runtime_call--------------------
1847 // Reading and setting the memory state is way conservative here.
1848 // The real problem is that I am not doing real Type analysis on memory,
1849 // so I cannot distinguish card mark stores from other stores.  Across a GC
1850 // point the Store Barrier and the card mark memory has to agree.  I cannot
1851 // have a card mark store and its barrier split across the GC point from
1852 // either above or below.  Here I get that to happen by reading ALL of memory.
1853 // A better answer would be to separate out card marks from other memory.
1854 // For now, return the input memory state, so that it can be reused
1855 // after the call, if this call has restricted memory effects.
1856 Node* GraphKit::set_predefined_input_for_runtime_call(SafePointNode* call, Node* narrow_mem) {
1857   // Set fixed predefined input arguments
1858   Node* memory = reset_memory();
1859   Node* m = narrow_mem == nullptr ? memory : narrow_mem;
1860   call->init_req( TypeFunc::Control,   control()  );
1861   call->init_req( TypeFunc::I_O,       top()      ); // does no i/o
1862   call->init_req( TypeFunc::Memory,    m          ); // may gc ptrs

1913     if (use->is_MergeMem()) {
1914       wl.push(use);
1915     }
1916   }
1917 }
1918 
1919 // Replace the call with the current state of the kit.
1920 void GraphKit::replace_call(CallNode* call, Node* result, bool do_replaced_nodes) {
1921   JVMState* ejvms = nullptr;
1922   if (has_exceptions()) {
1923     ejvms = transfer_exceptions_into_jvms();
1924   }
1925 
1926   ReplacedNodes replaced_nodes = map()->replaced_nodes();
1927   ReplacedNodes replaced_nodes_exception;
1928   Node* ex_ctl = top();
1929 
1930   SafePointNode* final_state = stop();
1931 
1932   // Find all the needed outputs of this call
1933   CallProjections callprojs;
1934   call->extract_projections(&callprojs, true);
1935 
1936   Unique_Node_List wl;
1937   Node* init_mem = call->in(TypeFunc::Memory);
1938   Node* final_mem = final_state->in(TypeFunc::Memory);
1939   Node* final_ctl = final_state->in(TypeFunc::Control);
1940   Node* final_io = final_state->in(TypeFunc::I_O);
1941 
1942   // Replace all the old call edges with the edges from the inlining result
1943   if (callprojs.fallthrough_catchproj != nullptr) {
1944     C->gvn_replace_by(callprojs.fallthrough_catchproj, final_ctl);
1945   }
1946   if (callprojs.fallthrough_memproj != nullptr) {
1947     if (final_mem->is_MergeMem()) {
1948       // Parser's exits MergeMem was not transformed but may be optimized
1949       final_mem = _gvn.transform(final_mem);
1950     }
1951     C->gvn_replace_by(callprojs.fallthrough_memproj,   final_mem);
1952     add_mergemem_users_to_worklist(wl, final_mem);
1953   }
1954   if (callprojs.fallthrough_ioproj != nullptr) {
1955     C->gvn_replace_by(callprojs.fallthrough_ioproj,    final_io);
1956   }
1957 
1958   // Replace the result with the new result if it exists and is used
1959   if (callprojs.resproj != nullptr && result != nullptr) {
1960     C->gvn_replace_by(callprojs.resproj, result);




1961   }
1962 
1963   if (ejvms == nullptr) {
1964     // No exception edges to simply kill off those paths
1965     if (callprojs.catchall_catchproj != nullptr) {
1966       C->gvn_replace_by(callprojs.catchall_catchproj, C->top());
1967     }
1968     if (callprojs.catchall_memproj != nullptr) {
1969       C->gvn_replace_by(callprojs.catchall_memproj,   C->top());
1970     }
1971     if (callprojs.catchall_ioproj != nullptr) {
1972       C->gvn_replace_by(callprojs.catchall_ioproj,    C->top());
1973     }
1974     // Replace the old exception object with top
1975     if (callprojs.exobj != nullptr) {
1976       C->gvn_replace_by(callprojs.exobj, C->top());
1977     }
1978   } else {
1979     GraphKit ekit(ejvms);
1980 
1981     // Load my combined exception state into the kit, with all phis transformed:
1982     SafePointNode* ex_map = ekit.combine_and_pop_all_exception_states();
1983     replaced_nodes_exception = ex_map->replaced_nodes();
1984 
1985     Node* ex_oop = ekit.use_exception_state(ex_map);
1986 
1987     if (callprojs.catchall_catchproj != nullptr) {
1988       C->gvn_replace_by(callprojs.catchall_catchproj, ekit.control());
1989       ex_ctl = ekit.control();
1990     }
1991     if (callprojs.catchall_memproj != nullptr) {
1992       Node* ex_mem = ekit.reset_memory();
1993       C->gvn_replace_by(callprojs.catchall_memproj,   ex_mem);
1994       add_mergemem_users_to_worklist(wl, ex_mem);
1995     }
1996     if (callprojs.catchall_ioproj != nullptr) {
1997       C->gvn_replace_by(callprojs.catchall_ioproj,    ekit.i_o());
1998     }
1999 
2000     // Replace the old exception object with the newly created one
2001     if (callprojs.exobj != nullptr) {
2002       C->gvn_replace_by(callprojs.exobj, ex_oop);
2003     }
2004   }
2005 
2006   // Disconnect the call from the graph
2007   call->disconnect_inputs(C);
2008   C->gvn_replace_by(call, C->top());
2009 
2010   // Clean up any MergeMems that feed other MergeMems since the
2011   // optimizer doesn't like that.
2012   while (wl.size() > 0) {
2013     _gvn.transform(wl.pop());
2014   }
2015 
2016   if (callprojs.fallthrough_catchproj != nullptr && !final_ctl->is_top() && do_replaced_nodes) {
2017     replaced_nodes.apply(C, final_ctl);
2018   }
2019   if (!ex_ctl->is_top() && do_replaced_nodes) {
2020     replaced_nodes_exception.apply(C, ex_ctl);
2021   }
2022 }
2023 
2024 
2025 //------------------------------increment_counter------------------------------
2026 // for statistics: increment a VM counter by 1
2027 
2028 void GraphKit::increment_counter(address counter_addr) {
2029   Node* adr1 = makecon(TypeRawPtr::make(counter_addr));
2030   increment_counter(adr1);
2031 }
2032 
2033 void GraphKit::increment_counter(Node* counter_addr) {
2034   int adr_type = Compile::AliasIdxRaw;
2035   Node* ctrl = control();
2036   Node* cnt  = make_load(ctrl, counter_addr, TypeLong::LONG, T_LONG, adr_type, MemNode::unordered);

2195  *
2196  * @param n          node that the type applies to
2197  * @param exact_kls  type from profiling
2198  * @param maybe_null did profiling see null?
2199  *
2200  * @return           node with improved type
2201  */
2202 Node* GraphKit::record_profile_for_speculation(Node* n, ciKlass* exact_kls, ProfilePtrKind ptr_kind) {
2203   const Type* current_type = _gvn.type(n);
2204   assert(UseTypeSpeculation, "type speculation must be on");
2205 
2206   const TypePtr* speculative = current_type->speculative();
2207 
2208   // Should the klass from the profile be recorded in the speculative type?
2209   if (current_type->would_improve_type(exact_kls, jvms()->depth())) {
2210     const TypeKlassPtr* tklass = TypeKlassPtr::make(exact_kls, Type::trust_interfaces);
2211     const TypeOopPtr* xtype = tklass->as_instance_type();
2212     assert(xtype->klass_is_exact(), "Should be exact");
2213     // Any reason to believe n is not null (from this profiling or a previous one)?
2214     assert(ptr_kind != ProfileAlwaysNull, "impossible here");
2215     const TypePtr* ptr = (ptr_kind == ProfileMaybeNull && current_type->speculative_maybe_null()) ? TypePtr::BOTTOM : TypePtr::NOTNULL;
2216     // record the new speculative type's depth
2217     speculative = xtype->cast_to_ptr_type(ptr->ptr())->is_ptr();
2218     speculative = speculative->with_inline_depth(jvms()->depth());
2219   } else if (current_type->would_improve_ptr(ptr_kind)) {
2220     // Profiling report that null was never seen so we can change the
2221     // speculative type to non null ptr.
2222     if (ptr_kind == ProfileAlwaysNull) {
2223       speculative = TypePtr::NULL_PTR;
2224     } else {
2225       assert(ptr_kind == ProfileNeverNull, "nothing else is an improvement");
2226       const TypePtr* ptr = TypePtr::NOTNULL;
2227       if (speculative != nullptr) {
2228         speculative = speculative->cast_to_ptr_type(ptr->ptr())->is_ptr();
2229       } else {
2230         speculative = ptr;
2231       }
2232     }
2233   }
2234 
2235   if (speculative != current_type->speculative()) {
2236     // Build a type with a speculative type (what we think we know
2237     // about the type but will need a guard when we use it)
2238     const TypeOopPtr* spec_type = TypeOopPtr::make(TypePtr::BotPTR, Type::OffsetBot, TypeOopPtr::InstanceBot, speculative);
2239     // We're changing the type, we need a new CheckCast node to carry
2240     // the new type. The new type depends on the control: what
2241     // profiling tells us is only valid from here as far as we can
2242     // tell.
2243     Node* cast = new CheckCastPPNode(control(), n, current_type->remove_speculative()->join_speculative(spec_type));
2244     cast = _gvn.transform(cast);
2245     replace_in_map(n, cast);
2246     n = cast;
2247   }
2248 
2249   return n;
2250 }
2251 
2252 /**
2253  * Record profiling data from receiver profiling at an invoke with the
2254  * type system so that it can propagate it (speculation)
2255  *
2256  * @param n  receiver node
2257  *
2258  * @return   node with improved type
2259  */
2260 Node* GraphKit::record_profiled_receiver_for_speculation(Node* n) {
2261   if (!UseTypeSpeculation) {
2262     return n;
2263   }
2264   ciKlass* exact_kls = profile_has_unique_klass();
2265   ProfilePtrKind ptr_kind = ProfileMaybeNull;
2266   if ((java_bc() == Bytecodes::_checkcast ||
2267        java_bc() == Bytecodes::_instanceof ||
2268        java_bc() == Bytecodes::_aastore) &&
2269       method()->method_data()->is_mature()) {
2270     ciProfileData* data = method()->method_data()->bci_to_data(bci());
2271     if (data != nullptr) {
2272       if (!data->as_BitData()->null_seen()) {
2273         ptr_kind = ProfileNeverNull;







2274       } else {
2275         assert(data->is_ReceiverTypeData(), "bad profile data type");
2276         ciReceiverTypeData* call = (ciReceiverTypeData*)data->as_ReceiverTypeData();
2277         uint i = 0;
2278         for (; i < call->row_limit(); i++) {
2279           ciKlass* receiver = call->receiver(i);
2280           if (receiver != nullptr) {
2281             break;




2282           }

2283         }
2284         ptr_kind = (i == call->row_limit()) ? ProfileAlwaysNull : ProfileMaybeNull;
2285       }
2286     }
2287   }
2288   return record_profile_for_speculation(n, exact_kls, ptr_kind);
2289 }
2290 
2291 /**
2292  * Record profiling data from argument profiling at an invoke with the
2293  * type system so that it can propagate it (speculation)
2294  *
2295  * @param dest_method  target method for the call
2296  * @param bc           what invoke bytecode is this?
2297  */
2298 void GraphKit::record_profiled_arguments_for_speculation(ciMethod* dest_method, Bytecodes::Code bc) {
2299   if (!UseTypeSpeculation) {
2300     return;
2301   }
2302   const TypeFunc* tf    = TypeFunc::make(dest_method);
2303   int             nargs = tf->domain()->cnt() - TypeFunc::Parms;
2304   int skip = Bytecodes::has_receiver(bc) ? 1 : 0;
2305   for (int j = skip, i = 0; j < nargs && i < TypeProfileArgsLimit; j++) {
2306     const Type *targ = tf->domain()->field_at(j + TypeFunc::Parms);
2307     if (is_reference_type(targ->basic_type())) {
2308       ProfilePtrKind ptr_kind = ProfileMaybeNull;
2309       ciKlass* better_type = nullptr;
2310       if (method()->argument_profiled_type(bci(), i, better_type, ptr_kind)) {
2311         record_profile_for_speculation(argument(j), better_type, ptr_kind);
2312       }
2313       i++;
2314     }
2315   }
2316 }
2317 
2318 /**
2319  * Record profiling data from parameter profiling at an invoke with
2320  * the type system so that it can propagate it (speculation)
2321  */
2322 void GraphKit::record_profiled_parameters_for_speculation() {
2323   if (!UseTypeSpeculation) {
2324     return;
2325   }
2326   for (int i = 0, j = 0; i < method()->arg_size() ; i++) {

2340  * the type system so that it can propagate it (speculation)
2341  */
2342 void GraphKit::record_profiled_return_for_speculation() {
2343   if (!UseTypeSpeculation) {
2344     return;
2345   }
2346   ProfilePtrKind ptr_kind = ProfileMaybeNull;
2347   ciKlass* better_type = nullptr;
2348   if (method()->return_profiled_type(bci(), better_type, ptr_kind)) {
2349     // If profiling reports a single type for the return value,
2350     // feed it to the type system so it can propagate it as a
2351     // speculative type
2352     record_profile_for_speculation(stack(sp()-1), better_type, ptr_kind);
2353   }
2354 }
2355 
2356 void GraphKit::round_double_arguments(ciMethod* dest_method) {
2357   if (Matcher::strict_fp_requires_explicit_rounding) {
2358     // (Note:  TypeFunc::make has a cache that makes this fast.)
2359     const TypeFunc* tf    = TypeFunc::make(dest_method);
2360     int             nargs = tf->domain()->cnt() - TypeFunc::Parms;
2361     for (int j = 0; j < nargs; j++) {
2362       const Type *targ = tf->domain()->field_at(j + TypeFunc::Parms);
2363       if (targ->basic_type() == T_DOUBLE) {
2364         // If any parameters are doubles, they must be rounded before
2365         // the call, dprecision_rounding does gvn.transform
2366         Node *arg = argument(j);
2367         arg = dprecision_rounding(arg);
2368         set_argument(j, arg);
2369       }
2370     }
2371   }
2372 }
2373 
2374 // rounding for strict float precision conformance
2375 Node* GraphKit::precision_rounding(Node* n) {
2376   if (Matcher::strict_fp_requires_explicit_rounding) {
2377 #ifdef IA32
2378     if (UseSSE == 0) {
2379       return _gvn.transform(new RoundFloatNode(0, n));
2380     }
2381 #else
2382     Unimplemented();

2491                                   // The first null ends the list.
2492                                   Node* parm0, Node* parm1,
2493                                   Node* parm2, Node* parm3,
2494                                   Node* parm4, Node* parm5,
2495                                   Node* parm6, Node* parm7) {
2496   assert(call_addr != nullptr, "must not call null targets");
2497 
2498   // Slow-path call
2499   bool is_leaf = !(flags & RC_NO_LEAF);
2500   bool has_io  = (!is_leaf && !(flags & RC_NO_IO));
2501   if (call_name == nullptr) {
2502     assert(!is_leaf, "must supply name for leaf");
2503     call_name = OptoRuntime::stub_name(call_addr);
2504   }
2505   CallNode* call;
2506   if (!is_leaf) {
2507     call = new CallStaticJavaNode(call_type, call_addr, call_name, adr_type);
2508   } else if (flags & RC_NO_FP) {
2509     call = new CallLeafNoFPNode(call_type, call_addr, call_name, adr_type);
2510   } else  if (flags & RC_VECTOR){
2511     uint num_bits = call_type->range()->field_at(TypeFunc::Parms)->is_vect()->length_in_bytes() * BitsPerByte;
2512     call = new CallLeafVectorNode(call_type, call_addr, call_name, adr_type, num_bits);
2513   } else {
2514     call = new CallLeafNode(call_type, call_addr, call_name, adr_type);
2515   }
2516 
2517   // The following is similar to set_edges_for_java_call,
2518   // except that the memory effects of the call are restricted to AliasIdxRaw.
2519 
2520   // Slow path call has no side-effects, uses few values
2521   bool wide_in  = !(flags & RC_NARROW_MEM);
2522   bool wide_out = (C->get_alias_index(adr_type) == Compile::AliasIdxBot);
2523 
2524   Node* prev_mem = nullptr;
2525   if (wide_in) {
2526     prev_mem = set_predefined_input_for_runtime_call(call);
2527   } else {
2528     assert(!wide_out, "narrow in => narrow out");
2529     Node* narrow_mem = memory(adr_type);
2530     prev_mem = set_predefined_input_for_runtime_call(call, narrow_mem);
2531   }

2571 
2572   if (has_io) {
2573     set_i_o(_gvn.transform(new ProjNode(call, TypeFunc::I_O)));
2574   }
2575   return call;
2576 
2577 }
2578 
2579 // i2b
2580 Node* GraphKit::sign_extend_byte(Node* in) {
2581   Node* tmp = _gvn.transform(new LShiftINode(in, _gvn.intcon(24)));
2582   return _gvn.transform(new RShiftINode(tmp, _gvn.intcon(24)));
2583 }
2584 
2585 // i2s
2586 Node* GraphKit::sign_extend_short(Node* in) {
2587   Node* tmp = _gvn.transform(new LShiftINode(in, _gvn.intcon(16)));
2588   return _gvn.transform(new RShiftINode(tmp, _gvn.intcon(16)));
2589 }
2590 

2591 //------------------------------merge_memory-----------------------------------
2592 // Merge memory from one path into the current memory state.
2593 void GraphKit::merge_memory(Node* new_mem, Node* region, int new_path) {
2594   for (MergeMemStream mms(merged_memory(), new_mem->as_MergeMem()); mms.next_non_empty2(); ) {
2595     Node* old_slice = mms.force_memory();
2596     Node* new_slice = mms.memory2();
2597     if (old_slice != new_slice) {
2598       PhiNode* phi;
2599       if (old_slice->is_Phi() && old_slice->as_Phi()->region() == region) {
2600         if (mms.is_empty()) {
2601           // clone base memory Phi's inputs for this memory slice
2602           assert(old_slice == mms.base_memory(), "sanity");
2603           phi = PhiNode::make(region, nullptr, Type::MEMORY, mms.adr_type(C));
2604           _gvn.set_type(phi, Type::MEMORY);
2605           for (uint i = 1; i < phi->req(); i++) {
2606             phi->init_req(i, old_slice->in(i));
2607           }
2608         } else {
2609           phi = old_slice->as_Phi(); // Phi was generated already
2610         }

2867 
2868   // Now do a linear scan of the secondary super-klass array.  Again, no real
2869   // performance impact (too rare) but it's gotta be done.
2870   // Since the code is rarely used, there is no penalty for moving it
2871   // out of line, and it can only improve I-cache density.
2872   // The decision to inline or out-of-line this final check is platform
2873   // dependent, and is found in the AD file definition of PartialSubtypeCheck.
2874   Node* psc = gvn.transform(
2875     new PartialSubtypeCheckNode(*ctrl, subklass, superklass));
2876 
2877   IfNode *iff4 = gen_subtype_check_compare(*ctrl, psc, gvn.zerocon(T_OBJECT), BoolTest::ne, PROB_FAIR, gvn, T_ADDRESS);
2878   r_not_subtype->init_req(2, gvn.transform(new IfTrueNode (iff4)));
2879   r_ok_subtype ->init_req(3, gvn.transform(new IfFalseNode(iff4)));
2880 
2881   // Return false path; set default control to true path.
2882   *ctrl = gvn.transform(r_ok_subtype);
2883   return gvn.transform(r_not_subtype);
2884 }
2885 
2886 Node* GraphKit::gen_subtype_check(Node* obj_or_subklass, Node* superklass) {





2887   bool expand_subtype_check = C->post_loop_opts_phase() ||   // macro node expansion is over
2888                               ExpandSubTypeCheckAtParseTime; // forced expansion
2889   if (expand_subtype_check) {
2890     MergeMemNode* mem = merged_memory();
2891     Node* ctrl = control();
2892     Node* subklass = obj_or_subklass;
2893     if (!_gvn.type(obj_or_subklass)->isa_klassptr()) {
2894       subklass = load_object_klass(obj_or_subklass);
2895     }
2896 
2897     Node* n = Phase::gen_subtype_check(subklass, superklass, &ctrl, mem, _gvn, method(), bci());
2898     set_control(ctrl);
2899     return n;
2900   }
2901 
2902   Node* check = _gvn.transform(new SubTypeCheckNode(C, obj_or_subklass, superklass, method(), bci()));
2903   Node* bol = _gvn.transform(new BoolNode(check, BoolTest::eq));
2904   IfNode* iff = create_and_xform_if(control(), bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
2905   set_control(_gvn.transform(new IfTrueNode(iff)));
2906   return _gvn.transform(new IfFalseNode(iff));
2907 }
2908 
2909 // Profile-driven exact type check:
2910 Node* GraphKit::type_check_receiver(Node* receiver, ciKlass* klass,
2911                                     float prob,
2912                                     Node* *casted_receiver) {
2913   assert(!klass->is_interface(), "no exact type check on interfaces");
2914 











2915   const TypeKlassPtr* tklass = TypeKlassPtr::make(klass, Type::trust_interfaces);
2916   Node* recv_klass = load_object_klass(receiver);
2917   Node* want_klass = makecon(tklass);
2918   Node* cmp = _gvn.transform(new CmpPNode(recv_klass, want_klass));
2919   Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
2920   IfNode* iff = create_and_xform_if(control(), bol, prob, COUNT_UNKNOWN);
2921   set_control( _gvn.transform(new IfTrueNode (iff)));
2922   Node* fail = _gvn.transform(new IfFalseNode(iff));
2923 
2924   if (!stopped()) {
2925     const TypeOopPtr* receiver_type = _gvn.type(receiver)->isa_oopptr();
2926     const TypeOopPtr* recvx_type = tklass->as_instance_type();
2927     assert(recvx_type->klass_is_exact(), "");
2928 
2929     if (!receiver_type->higher_equal(recvx_type)) { // ignore redundant casts
2930       // Subsume downstream occurrences of receiver with a cast to
2931       // recv_xtype, since now we know what the type will be.
2932       Node* cast = new CheckCastPPNode(control(), receiver, recvx_type);
2933       (*casted_receiver) = _gvn.transform(cast);





2934       assert(!(*casted_receiver)->is_top(), "that path should be unreachable");
2935       // (User must make the replace_in_map call.)
2936     }
2937   }
2938 
2939   return fail;
2940 }
2941 











2942 //------------------------------subtype_check_receiver-------------------------
2943 Node* GraphKit::subtype_check_receiver(Node* receiver, ciKlass* klass,
2944                                        Node** casted_receiver) {
2945   const TypeKlassPtr* tklass = TypeKlassPtr::make(klass, Type::trust_interfaces)->try_improve();
2946   Node* want_klass = makecon(tklass);
2947 
2948   Node* slow_ctl = gen_subtype_check(receiver, want_klass);
2949 
2950   // Ignore interface type information until interface types are properly tracked.
2951   if (!stopped() && !klass->is_interface()) {
2952     const TypeOopPtr* receiver_type = _gvn.type(receiver)->isa_oopptr();
2953     const TypeOopPtr* recv_type = tklass->cast_to_exactness(false)->is_klassptr()->as_instance_type();
2954     if (!receiver_type->higher_equal(recv_type)) { // ignore redundant casts
2955       Node* cast = new CheckCastPPNode(control(), receiver, recv_type);
2956       (*casted_receiver) = _gvn.transform(cast);



2957     }
2958   }
2959 
2960   return slow_ctl;
2961 }
2962 
2963 //------------------------------seems_never_null-------------------------------
2964 // Use null_seen information if it is available from the profile.
2965 // If we see an unexpected null at a type check we record it and force a
2966 // recompile; the offending check will be recompiled to handle nulls.
2967 // If we see several offending BCIs, then all checks in the
2968 // method will be recompiled.
2969 bool GraphKit::seems_never_null(Node* obj, ciProfileData* data, bool& speculating) {
2970   speculating = !_gvn.type(obj)->speculative_maybe_null();
2971   Deoptimization::DeoptReason reason = Deoptimization::reason_null_check(speculating);
2972   if (UncommonNullCast               // Cutout for this technique
2973       && obj != null()               // And not the -Xcomp stupid case?
2974       && !too_many_traps(reason)
2975       ) {
2976     if (speculating) {

3045 
3046 //------------------------maybe_cast_profiled_receiver-------------------------
3047 // If the profile has seen exactly one type, narrow to exactly that type.
3048 // Subsequent type checks will always fold up.
3049 Node* GraphKit::maybe_cast_profiled_receiver(Node* not_null_obj,
3050                                              const TypeKlassPtr* require_klass,
3051                                              ciKlass* spec_klass,
3052                                              bool safe_for_replace) {
3053   if (!UseTypeProfile || !TypeProfileCasts) return nullptr;
3054 
3055   Deoptimization::DeoptReason reason = Deoptimization::reason_class_check(spec_klass != nullptr);
3056 
3057   // Make sure we haven't already deoptimized from this tactic.
3058   if (too_many_traps_or_recompiles(reason))
3059     return nullptr;
3060 
3061   // (No, this isn't a call, but it's enough like a virtual call
3062   // to use the same ciMethod accessor to get the profile info...)
3063   // If we have a speculative type use it instead of profiling (which
3064   // may not help us)
3065   ciKlass* exact_kls = spec_klass == nullptr ? profile_has_unique_klass() : spec_klass;













3066   if (exact_kls != nullptr) {// no cast failures here
3067     if (require_klass == nullptr ||
3068         C->static_subtype_check(require_klass, TypeKlassPtr::make(exact_kls, Type::trust_interfaces)) == Compile::SSC_always_true) {
3069       // If we narrow the type to match what the type profile sees or
3070       // the speculative type, we can then remove the rest of the
3071       // cast.
3072       // This is a win, even if the exact_kls is very specific,
3073       // because downstream operations, such as method calls,
3074       // will often benefit from the sharper type.
3075       Node* exact_obj = not_null_obj; // will get updated in place...
3076       Node* slow_ctl  = type_check_receiver(exact_obj, exact_kls, 1.0,
3077                                             &exact_obj);
3078       { PreserveJVMState pjvms(this);
3079         set_control(slow_ctl);
3080         uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
3081       }
3082       if (safe_for_replace) {
3083         replace_in_map(not_null_obj, exact_obj);
3084       }
3085       return exact_obj;

3175   // If not_null_obj is dead, only null-path is taken
3176   if (stopped()) {              // Doing instance-of on a null?
3177     set_control(null_ctl);
3178     return intcon(0);
3179   }
3180   region->init_req(_null_path, null_ctl);
3181   phi   ->init_req(_null_path, intcon(0)); // Set null path value
3182   if (null_ctl == top()) {
3183     // Do this eagerly, so that pattern matches like is_diamond_phi
3184     // will work even during parsing.
3185     assert(_null_path == PATH_LIMIT-1, "delete last");
3186     region->del_req(_null_path);
3187     phi   ->del_req(_null_path);
3188   }
3189 
3190   // Do we know the type check always succeed?
3191   bool known_statically = false;
3192   if (_gvn.type(superklass)->singleton()) {
3193     const TypeKlassPtr* superk = _gvn.type(superklass)->is_klassptr();
3194     const TypeKlassPtr* subk = _gvn.type(obj)->is_oopptr()->as_klass_type();
3195     if (subk->is_loaded()) {
3196       int static_res = C->static_subtype_check(superk, subk);
3197       known_statically = (static_res == Compile::SSC_always_true || static_res == Compile::SSC_always_false);
3198     }
3199   }
3200 
3201   if (!known_statically) {
3202     const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr();
3203     // We may not have profiling here or it may not help us. If we
3204     // have a speculative type use it to perform an exact cast.
3205     ciKlass* spec_obj_type = obj_type->speculative_type();
3206     if (spec_obj_type != nullptr || (ProfileDynamicTypes && data != nullptr)) {
3207       Node* cast_obj = maybe_cast_profiled_receiver(not_null_obj, nullptr, spec_obj_type, safe_for_replace);
3208       if (stopped()) {            // Profile disagrees with this path.
3209         set_control(null_ctl);    // Null is the only remaining possibility.
3210         return intcon(0);
3211       }
3212       if (cast_obj != nullptr) {
3213         not_null_obj = cast_obj;
3214       }
3215     }

3231   record_for_igvn(region);
3232 
3233   // If we know the type check always succeeds then we don't use the
3234   // profiling data at this bytecode. Don't lose it, feed it to the
3235   // type system as a speculative type.
3236   if (safe_for_replace) {
3237     Node* casted_obj = record_profiled_receiver_for_speculation(obj);
3238     replace_in_map(obj, casted_obj);
3239   }
3240 
3241   return _gvn.transform(phi);
3242 }
3243 
3244 //-------------------------------gen_checkcast---------------------------------
3245 // Generate a checkcast idiom.  Used by both the checkcast bytecode and the
3246 // array store bytecode.  Stack must be as-if BEFORE doing the bytecode so the
3247 // uncommon-trap paths work.  Adjust stack after this call.
3248 // If failure_control is supplied and not null, it is filled in with
3249 // the control edge for the cast failure.  Otherwise, an appropriate
3250 // uncommon trap or exception is thrown.
3251 Node* GraphKit::gen_checkcast(Node *obj, Node* superklass,
3252                               Node* *failure_control) {
3253   kill_dead_locals();           // Benefit all the uncommon traps
3254   const TypeKlassPtr *tk = _gvn.type(superklass)->is_klassptr()->try_improve();
3255   const TypeOopPtr *toop = tk->cast_to_exactness(false)->as_instance_type();


3256 
3257   // Fast cutout:  Check the case that the cast is vacuously true.
3258   // This detects the common cases where the test will short-circuit
3259   // away completely.  We do this before we perform the null check,
3260   // because if the test is going to turn into zero code, we don't
3261   // want a residual null check left around.  (Causes a slowdown,
3262   // for example, in some objArray manipulations, such as a[i]=a[j].)
3263   if (tk->singleton()) {
3264     const TypeOopPtr* objtp = _gvn.type(obj)->isa_oopptr();
3265     if (objtp != nullptr) {
3266       switch (C->static_subtype_check(tk, objtp->as_klass_type())) {







3267       case Compile::SSC_always_true:
3268         // If we know the type check always succeed then we don't use
3269         // the profiling data at this bytecode. Don't lose it, feed it
3270         // to the type system as a speculative type.
3271         return record_profiled_receiver_for_speculation(obj);






3272       case Compile::SSC_always_false:




3273         // It needs a null check because a null will *pass* the cast check.
3274         // A non-null value will always produce an exception.
3275         if (!objtp->maybe_null()) {
3276           bool is_aastore = (java_bc() == Bytecodes::_aastore);
3277           Deoptimization::DeoptReason reason = is_aastore ?
3278             Deoptimization::Reason_array_check : Deoptimization::Reason_class_check;
3279           builtin_throw(reason);
3280           return top();
3281         } else if (!too_many_traps_or_recompiles(Deoptimization::Reason_null_assert)) {
3282           return null_assert(obj);
3283         }
3284         break; // Fall through to full check
3285       default:
3286         break;
3287       }
3288     }
3289   }
3290 
3291   ciProfileData* data = nullptr;
3292   bool safe_for_replace = false;
3293   if (failure_control == nullptr) {        // use MDO in regular case only
3294     assert(java_bc() == Bytecodes::_aastore ||
3295            java_bc() == Bytecodes::_checkcast,
3296            "interpreter profiles type checks only for these BCs");
3297     data = method()->method_data()->bci_to_data(bci());
3298     safe_for_replace = true;

3299   }
3300 
3301   // Make the merge point
3302   enum { _obj_path = 1, _null_path, PATH_LIMIT };
3303   RegionNode* region = new RegionNode(PATH_LIMIT);
3304   Node*       phi    = new PhiNode(region, toop);



3305   C->set_has_split_ifs(true); // Has chance for split-if optimization
3306 
3307   // Use null-cast information if it is available
3308   bool speculative_not_null = false;
3309   bool never_see_null = ((failure_control == nullptr)  // regular case only
3310                          && seems_never_null(obj, data, speculative_not_null));
3311 







3312   // Null check; get casted pointer; set region slot 3
3313   Node* null_ctl = top();
3314   Node* not_null_obj = null_check_oop(obj, &null_ctl, never_see_null, safe_for_replace, speculative_not_null);






3315 
3316   // If not_null_obj is dead, only null-path is taken
3317   if (stopped()) {              // Doing instance-of on a null?
3318     set_control(null_ctl);



3319     return null();
3320   }
3321   region->init_req(_null_path, null_ctl);
3322   phi   ->init_req(_null_path, null());  // Set null path value
3323   if (null_ctl == top()) {
3324     // Do this eagerly, so that pattern matches like is_diamond_phi
3325     // will work even during parsing.
3326     assert(_null_path == PATH_LIMIT-1, "delete last");
3327     region->del_req(_null_path);
3328     phi   ->del_req(_null_path);
3329   }
3330 
3331   Node* cast_obj = nullptr;
3332   if (tk->klass_is_exact()) {
3333     // The following optimization tries to statically cast the speculative type of the object
3334     // (for example obtained during profiling) to the type of the superklass and then do a
3335     // dynamic check that the type of the object is what we expect. To work correctly
3336     // for checkcast and aastore the type of superklass should be exact.
3337     const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr();
3338     // We may not have profiling here or it may not help us. If we have
3339     // a speculative type use it to perform an exact cast.
3340     ciKlass* spec_obj_type = obj_type->speculative_type();
3341     if (spec_obj_type != nullptr || data != nullptr) {
3342       cast_obj = maybe_cast_profiled_receiver(not_null_obj, tk, spec_obj_type, safe_for_replace);
3343       if (cast_obj != nullptr) {
3344         if (failure_control != nullptr) // failure is now impossible
3345           (*failure_control) = top();
3346         // adjust the type of the phi to the exact klass:
3347         phi->raise_bottom_type(_gvn.type(cast_obj)->meet_speculative(TypePtr::NULL_PTR));
3348       }
3349     }
3350   }
3351 
3352   if (cast_obj == nullptr) {
3353     // Generate the subtype check
3354     Node* not_subtype_ctrl = gen_subtype_check(not_null_obj, superklass );
3355 
3356     // Plug in success path into the merge
3357     cast_obj = _gvn.transform(new CheckCastPPNode(control(), not_null_obj, toop));
3358     // Failure path ends in uncommon trap (or may be dead - failure impossible)
3359     if (failure_control == nullptr) {
3360       if (not_subtype_ctrl != top()) { // If failure is possible
3361         PreserveJVMState pjvms(this);
3362         set_control(not_subtype_ctrl);






3363         bool is_aastore = (java_bc() == Bytecodes::_aastore);
3364         Deoptimization::DeoptReason reason = is_aastore ?
3365           Deoptimization::Reason_array_check : Deoptimization::Reason_class_check;
3366         builtin_throw(reason);
3367       }
3368     } else {
3369       (*failure_control) = not_subtype_ctrl;
3370     }
3371   }
3372 
3373   region->init_req(_obj_path, control());
3374   phi   ->init_req(_obj_path, cast_obj);
3375 
3376   // A merge of null or Casted-NotNull obj
3377   Node* res = _gvn.transform(phi);
3378 
3379   // Note I do NOT always 'replace_in_map(obj,result)' here.
3380   //  if( tk->klass()->can_be_primary_super()  )
3381     // This means that if I successfully store an Object into an array-of-String
3382     // I 'forget' that the Object is really now known to be a String.  I have to
3383     // do this because we don't have true union types for interfaces - if I store
3384     // a Baz into an array-of-Interface and then tell the optimizer it's an
3385     // Interface, I forget that it's also a Baz and cannot do Baz-like field
3386     // references to it.  FIX THIS WHEN UNION TYPES APPEAR!
3387   //  replace_in_map( obj, res );
3388 
3389   // Return final merged results
3390   set_control( _gvn.transform(region) );
3391   record_for_igvn(region);
3392 
3393   return record_profiled_receiver_for_speculation(res);


























































































































3394 }
3395 
3396 //------------------------------next_monitor-----------------------------------
3397 // What number should be given to the next monitor?
3398 int GraphKit::next_monitor() {
3399   int current = jvms()->monitor_depth()* C->sync_stack_slots();
3400   int next = current + C->sync_stack_slots();
3401   // Keep the toplevel high water mark current:
3402   if (C->fixed_slots() < next)  C->set_fixed_slots(next);
3403   return current;
3404 }
3405 
3406 //------------------------------insert_mem_bar---------------------------------
3407 // Memory barrier to avoid floating things around
3408 // The membar serves as a pinch point between both control and all memory slices.
3409 Node* GraphKit::insert_mem_bar(int opcode, Node* precedent) {
3410   MemBarNode* mb = MemBarNode::make(C, opcode, Compile::AliasIdxBot, precedent);
3411   mb->init_req(TypeFunc::Control, control());
3412   mb->init_req(TypeFunc::Memory,  reset_memory());
3413   Node* membar = _gvn.transform(mb);

3441   }
3442   Node* membar = _gvn.transform(mb);
3443   set_control(_gvn.transform(new ProjNode(membar, TypeFunc::Control)));
3444   if (alias_idx == Compile::AliasIdxBot) {
3445     merged_memory()->set_base_memory(_gvn.transform(new ProjNode(membar, TypeFunc::Memory)));
3446   } else {
3447     set_memory(_gvn.transform(new ProjNode(membar, TypeFunc::Memory)),alias_idx);
3448   }
3449   return membar;
3450 }
3451 
3452 //------------------------------shared_lock------------------------------------
3453 // Emit locking code.
3454 FastLockNode* GraphKit::shared_lock(Node* obj) {
3455   // bci is either a monitorenter bc or InvocationEntryBci
3456   // %%% SynchronizationEntryBCI is redundant; use InvocationEntryBci in interfaces
3457   assert(SynchronizationEntryBCI == InvocationEntryBci, "");
3458 
3459   if( !GenerateSynchronizationCode )
3460     return nullptr;                // Not locking things?

3461   if (stopped())                // Dead monitor?
3462     return nullptr;
3463 
3464   assert(dead_locals_are_killed(), "should kill locals before sync. point");
3465 
3466   // Box the stack location
3467   Node* box = _gvn.transform(new BoxLockNode(next_monitor()));
3468   Node* mem = reset_memory();
3469 
3470   FastLockNode * flock = _gvn.transform(new FastLockNode(0, obj, box) )->as_FastLock();
3471 
3472   // Create the rtm counters for this fast lock if needed.
3473   flock->create_rtm_lock_counter(sync_jvms()); // sync_jvms used to get current bci
3474 
3475   // Add monitor to debug info for the slow path.  If we block inside the
3476   // slow path and de-opt, we need the monitor hanging around
3477   map()->push_monitor( flock );
3478 
3479   const TypeFunc *tf = LockNode::lock_type();
3480   LockNode *lock = new LockNode(C, tf);

3509   }
3510 #endif
3511 
3512   return flock;
3513 }
3514 
3515 
3516 //------------------------------shared_unlock----------------------------------
3517 // Emit unlocking code.
3518 void GraphKit::shared_unlock(Node* box, Node* obj) {
3519   // bci is either a monitorenter bc or InvocationEntryBci
3520   // %%% SynchronizationEntryBCI is redundant; use InvocationEntryBci in interfaces
3521   assert(SynchronizationEntryBCI == InvocationEntryBci, "");
3522 
3523   if( !GenerateSynchronizationCode )
3524     return;
3525   if (stopped()) {               // Dead monitor?
3526     map()->pop_monitor();        // Kill monitor from debug info
3527     return;
3528   }

3529 
3530   // Memory barrier to avoid floating things down past the locked region
3531   insert_mem_bar(Op_MemBarReleaseLock);
3532 
3533   const TypeFunc *tf = OptoRuntime::complete_monitor_exit_Type();
3534   UnlockNode *unlock = new UnlockNode(C, tf);
3535 #ifdef ASSERT
3536   unlock->set_dbg_jvms(sync_jvms());
3537 #endif
3538   uint raw_idx = Compile::AliasIdxRaw;
3539   unlock->init_req( TypeFunc::Control, control() );
3540   unlock->init_req( TypeFunc::Memory , memory(raw_idx) );
3541   unlock->init_req( TypeFunc::I_O    , top() )     ;   // does no i/o
3542   unlock->init_req( TypeFunc::FramePtr, frameptr() );
3543   unlock->init_req( TypeFunc::ReturnAdr, top() );
3544 
3545   unlock->init_req(TypeFunc::Parms + 0, obj);
3546   unlock->init_req(TypeFunc::Parms + 1, box);
3547   unlock = _gvn.transform(unlock)->as_Unlock();
3548 
3549   Node* mem = reset_memory();
3550 
3551   // unlock has no side-effects, sets few values
3552   set_predefined_output_for_runtime_call(unlock, mem, TypeRawPtr::BOTTOM);
3553 
3554   // Kill monitor from debug info
3555   map()->pop_monitor( );
3556 }
3557 
3558 //-------------------------------get_layout_helper-----------------------------
3559 // If the given klass is a constant or known to be an array,
3560 // fetch the constant layout helper value into constant_value
3561 // and return null.  Otherwise, load the non-constant
3562 // layout helper value, and return the node which represents it.
3563 // This two-faced routine is useful because allocation sites
3564 // almost always feature constant types.
3565 Node* GraphKit::get_layout_helper(Node* klass_node, jint& constant_value) {
3566   const TypeKlassPtr* klass_t = _gvn.type(klass_node)->isa_klassptr();
3567   if (!StressReflectiveCode && klass_t != nullptr) {
3568     bool xklass = klass_t->klass_is_exact();
3569     if (xklass || (klass_t->isa_aryklassptr() && klass_t->is_aryklassptr()->elem() != Type::BOTTOM)) {







3570       jint lhelper;
3571       if (klass_t->isa_aryklassptr()) {
3572         BasicType elem = klass_t->as_instance_type()->isa_aryptr()->elem()->array_element_basic_type();


3573         if (is_reference_type(elem, true)) {
3574           elem = T_OBJECT;
3575         }
3576         lhelper = Klass::array_layout_helper(elem);
3577       } else {
3578         lhelper = klass_t->is_instklassptr()->exact_klass()->layout_helper();
3579       }
3580       if (lhelper != Klass::_lh_neutral_value) {
3581         constant_value = lhelper;
3582         return (Node*) nullptr;
3583       }
3584     }
3585   }
3586   constant_value = Klass::_lh_neutral_value;  // put in a known value
3587   Node* lhp = basic_plus_adr(klass_node, klass_node, in_bytes(Klass::layout_helper_offset()));
3588   return make_load(nullptr, lhp, TypeInt::INT, T_INT, MemNode::unordered);
3589 }
3590 
3591 // We just put in an allocate/initialize with a big raw-memory effect.
3592 // Hook selected additional alias categories on the initialization.
3593 static void hook_memory_on_init(GraphKit& kit, int alias_idx,
3594                                 MergeMemNode* init_in_merge,
3595                                 Node* init_out_raw) {
3596   DEBUG_ONLY(Node* init_in_raw = init_in_merge->base_memory());
3597   assert(init_in_merge->memory_at(alias_idx) == init_in_raw, "");
3598 
3599   Node* prevmem = kit.memory(alias_idx);
3600   init_in_merge->set_memory_at(alias_idx, prevmem);
3601   kit.set_memory(init_out_raw, alias_idx);


3602 }
3603 
3604 //---------------------------set_output_for_allocation-------------------------
3605 Node* GraphKit::set_output_for_allocation(AllocateNode* alloc,
3606                                           const TypeOopPtr* oop_type,
3607                                           bool deoptimize_on_exception) {
3608   int rawidx = Compile::AliasIdxRaw;
3609   alloc->set_req( TypeFunc::FramePtr, frameptr() );
3610   add_safepoint_edges(alloc);
3611   Node* allocx = _gvn.transform(alloc);
3612   set_control( _gvn.transform(new ProjNode(allocx, TypeFunc::Control) ) );
3613   // create memory projection for i_o
3614   set_memory ( _gvn.transform( new ProjNode(allocx, TypeFunc::Memory, true) ), rawidx );
3615   make_slow_call_ex(allocx, env()->Throwable_klass(), true, deoptimize_on_exception);
3616 
3617   // create a memory projection as for the normal control path
3618   Node* malloc = _gvn.transform(new ProjNode(allocx, TypeFunc::Memory));
3619   set_memory(malloc, rawidx);
3620 
3621   // a normal slow-call doesn't change i_o, but an allocation does
3622   // we create a separate i_o projection for the normal control path
3623   set_i_o(_gvn.transform( new ProjNode(allocx, TypeFunc::I_O, false) ) );
3624   Node* rawoop = _gvn.transform( new ProjNode(allocx, TypeFunc::Parms) );
3625 
3626   // put in an initialization barrier
3627   InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, rawidx,
3628                                                  rawoop)->as_Initialize();
3629   assert(alloc->initialization() == init,  "2-way macro link must work");
3630   assert(init ->allocation()     == alloc, "2-way macro link must work");
3631   {
3632     // Extract memory strands which may participate in the new object's
3633     // initialization, and source them from the new InitializeNode.
3634     // This will allow us to observe initializations when they occur,
3635     // and link them properly (as a group) to the InitializeNode.
3636     assert(init->in(InitializeNode::Memory) == malloc, "");
3637     MergeMemNode* minit_in = MergeMemNode::make(malloc);
3638     init->set_req(InitializeNode::Memory, minit_in);
3639     record_for_igvn(minit_in); // fold it up later, if possible

3640     Node* minit_out = memory(rawidx);
3641     assert(minit_out->is_Proj() && minit_out->in(0) == init, "");
3642     // Add an edge in the MergeMem for the header fields so an access
3643     // to one of those has correct memory state
3644     set_memory(minit_out, C->get_alias_index(oop_type->add_offset(oopDesc::mark_offset_in_bytes())));
3645     set_memory(minit_out, C->get_alias_index(oop_type->add_offset(oopDesc::klass_offset_in_bytes())));
3646     if (oop_type->isa_aryptr()) {
3647       const TypePtr* telemref = oop_type->add_offset(Type::OffsetBot);
3648       int            elemidx  = C->get_alias_index(telemref);
3649       hook_memory_on_init(*this, elemidx, minit_in, minit_out);

























3650     } else if (oop_type->isa_instptr()) {

3651       ciInstanceKlass* ik = oop_type->is_instptr()->instance_klass();
3652       for (int i = 0, len = ik->nof_nonstatic_fields(); i < len; i++) {
3653         ciField* field = ik->nonstatic_field_at(i);
3654         if (field->offset_in_bytes() >= TrackedInitializationLimit * HeapWordSize)
3655           continue;  // do not bother to track really large numbers of fields
3656         // Find (or create) the alias category for this field:
3657         int fieldidx = C->alias_type(field)->index();
3658         hook_memory_on_init(*this, fieldidx, minit_in, minit_out);
3659       }
3660     }
3661   }
3662 
3663   // Cast raw oop to the real thing...
3664   Node* javaoop = new CheckCastPPNode(control(), rawoop, oop_type);
3665   javaoop = _gvn.transform(javaoop);
3666   C->set_recent_alloc(control(), javaoop);
3667   assert(just_allocated_object(control()) == javaoop, "just allocated");
3668 
3669 #ifdef ASSERT
3670   { // Verify that the AllocateNode::Ideal_allocation recognizers work:

3681       assert(alloc->in(AllocateNode::ALength)->is_top(), "no length, please");
3682     }
3683   }
3684 #endif //ASSERT
3685 
3686   return javaoop;
3687 }
3688 
3689 //---------------------------new_instance--------------------------------------
3690 // This routine takes a klass_node which may be constant (for a static type)
3691 // or may be non-constant (for reflective code).  It will work equally well
3692 // for either, and the graph will fold nicely if the optimizer later reduces
3693 // the type to a constant.
3694 // The optional arguments are for specialized use by intrinsics:
3695 //  - If 'extra_slow_test' if not null is an extra condition for the slow-path.
3696 //  - If 'return_size_val', report the total object size to the caller.
3697 //  - deoptimize_on_exception controls how Java exceptions are handled (rethrow vs deoptimize)
3698 Node* GraphKit::new_instance(Node* klass_node,
3699                              Node* extra_slow_test,
3700                              Node* *return_size_val,
3701                              bool deoptimize_on_exception) {

3702   // Compute size in doublewords
3703   // The size is always an integral number of doublewords, represented
3704   // as a positive bytewise size stored in the klass's layout_helper.
3705   // The layout_helper also encodes (in a low bit) the need for a slow path.
3706   jint  layout_con = Klass::_lh_neutral_value;
3707   Node* layout_val = get_layout_helper(klass_node, layout_con);
3708   int   layout_is_con = (layout_val == nullptr);
3709 
3710   if (extra_slow_test == nullptr)  extra_slow_test = intcon(0);
3711   // Generate the initial go-slow test.  It's either ALWAYS (return a
3712   // Node for 1) or NEVER (return a null) or perhaps (in the reflective
3713   // case) a computed value derived from the layout_helper.
3714   Node* initial_slow_test = nullptr;
3715   if (layout_is_con) {
3716     assert(!StressReflectiveCode, "stress mode does not use these paths");
3717     bool must_go_slow = Klass::layout_helper_needs_slow_path(layout_con);
3718     initial_slow_test = must_go_slow ? intcon(1) : extra_slow_test;
3719   } else {   // reflective case
3720     // This reflective path is used by Unsafe.allocateInstance.
3721     // (It may be stress-tested by specifying StressReflectiveCode.)
3722     // Basically, we want to get into the VM is there's an illegal argument.
3723     Node* bit = intcon(Klass::_lh_instance_slow_path_bit);
3724     initial_slow_test = _gvn.transform( new AndINode(layout_val, bit) );
3725     if (extra_slow_test != intcon(0)) {
3726       initial_slow_test = _gvn.transform( new OrINode(initial_slow_test, extra_slow_test) );
3727     }
3728     // (Macro-expander will further convert this to a Bool, if necessary.)

3739 
3740     // Clear the low bits to extract layout_helper_size_in_bytes:
3741     assert((int)Klass::_lh_instance_slow_path_bit < BytesPerLong, "clear bit");
3742     Node* mask = MakeConX(~ (intptr_t)right_n_bits(LogBytesPerLong));
3743     size = _gvn.transform( new AndXNode(size, mask) );
3744   }
3745   if (return_size_val != nullptr) {
3746     (*return_size_val) = size;
3747   }
3748 
3749   // This is a precise notnull oop of the klass.
3750   // (Actually, it need not be precise if this is a reflective allocation.)
3751   // It's what we cast the result to.
3752   const TypeKlassPtr* tklass = _gvn.type(klass_node)->isa_klassptr();
3753   if (!tklass)  tklass = TypeInstKlassPtr::OBJECT;
3754   const TypeOopPtr* oop_type = tklass->as_instance_type();
3755 
3756   // Now generate allocation code
3757 
3758   // The entire memory state is needed for slow path of the allocation
3759   // since GC and deoptimization can happened.
3760   Node *mem = reset_memory();
3761   set_all_memory(mem); // Create new memory state
3762 
3763   AllocateNode* alloc = new AllocateNode(C, AllocateNode::alloc_type(Type::TOP),
3764                                          control(), mem, i_o(),
3765                                          size, klass_node,
3766                                          initial_slow_test);
3767 
3768   return set_output_for_allocation(alloc, oop_type, deoptimize_on_exception);
3769 }
3770 
3771 //-------------------------------new_array-------------------------------------
3772 // helper for both newarray and anewarray
3773 // The 'length' parameter is (obviously) the length of the array.
3774 // The optional arguments are for specialized use by intrinsics:
3775 //  - If 'return_size_val', report the non-padded array size (sum of header size
3776 //    and array body) to the caller.
3777 //  - deoptimize_on_exception controls how Java exceptions are handled (rethrow vs deoptimize)
3778 Node* GraphKit::new_array(Node* klass_node,     // array klass (maybe variable)
3779                           Node* length,         // number of array elements
3780                           int   nargs,          // number of arguments to push back for uncommon trap
3781                           Node* *return_size_val,
3782                           bool deoptimize_on_exception) {
3783   jint  layout_con = Klass::_lh_neutral_value;
3784   Node* layout_val = get_layout_helper(klass_node, layout_con);
3785   int   layout_is_con = (layout_val == nullptr);
3786 
3787   if (!layout_is_con && !StressReflectiveCode &&
3788       !too_many_traps(Deoptimization::Reason_class_check)) {
3789     // This is a reflective array creation site.
3790     // Optimistically assume that it is a subtype of Object[],
3791     // so that we can fold up all the address arithmetic.
3792     layout_con = Klass::array_layout_helper(T_OBJECT);
3793     Node* cmp_lh = _gvn.transform( new CmpINode(layout_val, intcon(layout_con)) );
3794     Node* bol_lh = _gvn.transform( new BoolNode(cmp_lh, BoolTest::eq) );
3795     { BuildCutout unless(this, bol_lh, PROB_MAX);
3796       inc_sp(nargs);
3797       uncommon_trap(Deoptimization::Reason_class_check,
3798                     Deoptimization::Action_maybe_recompile);
3799     }
3800     layout_val = nullptr;
3801     layout_is_con = true;
3802   }
3803 
3804   // Generate the initial go-slow test.  Make sure we do not overflow
3805   // if length is huge (near 2Gig) or negative!  We do not need
3806   // exact double-words here, just a close approximation of needed
3807   // double-words.  We can't add any offset or rounding bits, lest we
3808   // take a size -1 of bytes and make it positive.  Use an unsigned
3809   // compare, so negative sizes look hugely positive.
3810   int fast_size_limit = FastAllocateSizeLimit;
3811   if (layout_is_con) {
3812     assert(!StressReflectiveCode, "stress mode does not use these paths");
3813     // Increase the size limit if we have exact knowledge of array type.
3814     int log2_esize = Klass::layout_helper_log2_element_size(layout_con);
3815     fast_size_limit <<= (LogBytesPerLong - log2_esize);
3816   }
3817 
3818   Node* initial_slow_cmp  = _gvn.transform( new CmpUNode( length, intcon( fast_size_limit ) ) );
3819   Node* initial_slow_test = _gvn.transform( new BoolNode( initial_slow_cmp, BoolTest::gt ) );
3820 
3821   // --- Size Computation ---
3822   // array_size = round_to_heap(array_header + (length << elem_shift));
3823   // where round_to_heap(x) == align_to(x, MinObjAlignmentInBytes)
3824   // and align_to(x, y) == ((x + y-1) & ~(y-1))
3825   // The rounding mask is strength-reduced, if possible.
3826   int round_mask = MinObjAlignmentInBytes - 1;
3827   Node* header_size = nullptr;
3828   // (T_BYTE has the weakest alignment and size restrictions...)
3829   if (layout_is_con) {
3830     int       hsize  = Klass::layout_helper_header_size(layout_con);
3831     int       eshift = Klass::layout_helper_log2_element_size(layout_con);

3832     if ((round_mask & ~right_n_bits(eshift)) == 0)
3833       round_mask = 0;  // strength-reduce it if it goes away completely
3834     assert((hsize & right_n_bits(eshift)) == 0, "hsize is pre-rounded");
3835     int header_size_min = arrayOopDesc::base_offset_in_bytes(T_BYTE);
3836     assert(header_size_min <= hsize, "generic minimum is smallest");
3837     header_size = intcon(hsize);
3838   } else {
3839     Node* hss   = intcon(Klass::_lh_header_size_shift);
3840     Node* hsm   = intcon(Klass::_lh_header_size_mask);
3841     header_size = _gvn.transform(new URShiftINode(layout_val, hss));
3842     header_size = _gvn.transform(new AndINode(header_size, hsm));
3843   }
3844 
3845   Node* elem_shift = nullptr;
3846   if (layout_is_con) {
3847     int eshift = Klass::layout_helper_log2_element_size(layout_con);
3848     if (eshift != 0)
3849       elem_shift = intcon(eshift);
3850   } else {
3851     // There is no need to mask or shift this value.
3852     // The semantics of LShiftINode include an implicit mask to 0x1F.
3853     assert(Klass::_lh_log2_element_size_shift == 0, "use shift in place");
3854     elem_shift = layout_val;

3901   }
3902   Node* non_rounded_size = _gvn.transform(new AddXNode(headerx, abody));
3903 
3904   if (return_size_val != nullptr) {
3905     // This is the size
3906     (*return_size_val) = non_rounded_size;
3907   }
3908 
3909   Node* size = non_rounded_size;
3910   if (round_mask != 0) {
3911     Node* mask1 = MakeConX(round_mask);
3912     size = _gvn.transform(new AddXNode(size, mask1));
3913     Node* mask2 = MakeConX(~round_mask);
3914     size = _gvn.transform(new AndXNode(size, mask2));
3915   }
3916   // else if round_mask == 0, the size computation is self-rounding
3917 
3918   // Now generate allocation code
3919 
3920   // The entire memory state is needed for slow path of the allocation
3921   // since GC and deoptimization can happened.
3922   Node *mem = reset_memory();
3923   set_all_memory(mem); // Create new memory state
3924 
3925   if (initial_slow_test->is_Bool()) {
3926     // Hide it behind a CMoveI, or else PhaseIdealLoop::split_up will get sick.
3927     initial_slow_test = initial_slow_test->as_Bool()->as_int_value(&_gvn);
3928   }
3929 
3930   const TypeOopPtr* ary_type = _gvn.type(klass_node)->is_klassptr()->as_instance_type();



























































3931   Node* valid_length_test = _gvn.intcon(1);
3932   if (ary_type->isa_aryptr()) {
3933     BasicType bt = ary_type->isa_aryptr()->elem()->array_element_basic_type();
3934     jint max = TypeAryPtr::max_array_length(bt);
3935     Node* valid_length_cmp  = _gvn.transform(new CmpUNode(length, intcon(max)));
3936     valid_length_test = _gvn.transform(new BoolNode(valid_length_cmp, BoolTest::le));
3937   }
3938 
3939   // Create the AllocateArrayNode and its result projections
3940   AllocateArrayNode* alloc
3941     = new AllocateArrayNode(C, AllocateArrayNode::alloc_type(TypeInt::INT),
3942                             control(), mem, i_o(),
3943                             size, klass_node,
3944                             initial_slow_test,
3945                             length, valid_length_test);
3946 
3947   // Cast to correct type.  Note that the klass_node may be constant or not,
3948   // and in the latter case the actual array type will be inexact also.
3949   // (This happens via a non-constant argument to inline_native_newArray.)
3950   // In any case, the value of klass_node provides the desired array type.
3951   const TypeInt* length_type = _gvn.find_int_type(length);
3952   if (ary_type->isa_aryptr() && length_type != nullptr) {
3953     // Try to get a better type than POS for the size
3954     ary_type = ary_type->is_aryptr()->cast_to_size(length_type);
3955   }
3956 
3957   Node* javaoop = set_output_for_allocation(alloc, ary_type, deoptimize_on_exception);
3958 
3959   array_ideal_length(alloc, ary_type, true);
3960   return javaoop;
3961 }
3962 
3963 // The following "Ideal_foo" functions are placed here because they recognize
3964 // the graph shapes created by the functions immediately above.
3965 
3966 //---------------------------Ideal_allocation----------------------------------

4073   set_all_memory(ideal.merged_memory());
4074   set_i_o(ideal.i_o());
4075   set_control(ideal.ctrl());
4076 }
4077 
4078 void GraphKit::final_sync(IdealKit& ideal) {
4079   // Final sync IdealKit and graphKit.
4080   sync_kit(ideal);
4081 }
4082 
4083 Node* GraphKit::load_String_length(Node* str, bool set_ctrl) {
4084   Node* len = load_array_length(load_String_value(str, set_ctrl));
4085   Node* coder = load_String_coder(str, set_ctrl);
4086   // Divide length by 2 if coder is UTF16
4087   return _gvn.transform(new RShiftINode(len, coder));
4088 }
4089 
4090 Node* GraphKit::load_String_value(Node* str, bool set_ctrl) {
4091   int value_offset = java_lang_String::value_offset();
4092   const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4093                                                      false, nullptr, 0);
4094   const TypePtr* value_field_type = string_type->add_offset(value_offset);
4095   const TypeAryPtr* value_type = TypeAryPtr::make(TypePtr::NotNull,
4096                                                   TypeAry::make(TypeInt::BYTE, TypeInt::POS),
4097                                                   ciTypeArrayKlass::make(T_BYTE), true, 0);
4098   Node* p = basic_plus_adr(str, str, value_offset);
4099   Node* load = access_load_at(str, p, value_field_type, value_type, T_OBJECT,
4100                               IN_HEAP | (set_ctrl ? C2_CONTROL_DEPENDENT_LOAD : 0) | MO_UNORDERED);
4101   return load;
4102 }
4103 
4104 Node* GraphKit::load_String_coder(Node* str, bool set_ctrl) {
4105   if (!CompactStrings) {
4106     return intcon(java_lang_String::CODER_UTF16);
4107   }
4108   int coder_offset = java_lang_String::coder_offset();
4109   const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4110                                                      false, nullptr, 0);
4111   const TypePtr* coder_field_type = string_type->add_offset(coder_offset);
4112 
4113   Node* p = basic_plus_adr(str, str, coder_offset);
4114   Node* load = access_load_at(str, p, coder_field_type, TypeInt::BYTE, T_BYTE,
4115                               IN_HEAP | (set_ctrl ? C2_CONTROL_DEPENDENT_LOAD : 0) | MO_UNORDERED);
4116   return load;
4117 }
4118 
4119 void GraphKit::store_String_value(Node* str, Node* value) {
4120   int value_offset = java_lang_String::value_offset();
4121   const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4122                                                      false, nullptr, 0);
4123   const TypePtr* value_field_type = string_type->add_offset(value_offset);
4124 
4125   access_store_at(str,  basic_plus_adr(str, value_offset), value_field_type,
4126                   value, TypeAryPtr::BYTES, T_OBJECT, IN_HEAP | MO_UNORDERED);
4127 }
4128 
4129 void GraphKit::store_String_coder(Node* str, Node* value) {
4130   int coder_offset = java_lang_String::coder_offset();
4131   const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4132                                                      false, nullptr, 0);
4133   const TypePtr* coder_field_type = string_type->add_offset(coder_offset);
4134 
4135   access_store_at(str, basic_plus_adr(str, coder_offset), coder_field_type,
4136                   value, TypeInt::BYTE, T_BYTE, IN_HEAP | MO_UNORDERED);
4137 }
4138 
4139 // Capture src and dst memory state with a MergeMemNode
4140 Node* GraphKit::capture_memory(const TypePtr* src_type, const TypePtr* dst_type) {
4141   if (src_type == dst_type) {
4142     // Types are equal, we don't need a MergeMemNode
4143     return memory(src_type);
4144   }
4145   MergeMemNode* merge = MergeMemNode::make(map()->memory());
4146   record_for_igvn(merge); // fold it up later, if possible
4147   int src_idx = C->get_alias_index(src_type);
4148   int dst_idx = C->get_alias_index(dst_type);
4149   merge->set_memory_at(src_idx, memory(src_idx));
4150   merge->set_memory_at(dst_idx, memory(dst_idx));
4151   return merge;
4152 }

4225   i_char->init_req(2, AddI(i_char, intcon(2)));
4226 
4227   set_control(IfFalse(iff));
4228   set_memory(st, TypeAryPtr::BYTES);
4229 }
4230 
4231 Node* GraphKit::make_constant_from_field(ciField* field, Node* obj) {
4232   if (!field->is_constant()) {
4233     return nullptr; // Field not marked as constant.
4234   }
4235   ciInstance* holder = nullptr;
4236   if (!field->is_static()) {
4237     ciObject* const_oop = obj->bottom_type()->is_oopptr()->const_oop();
4238     if (const_oop != nullptr && const_oop->is_instance()) {
4239       holder = const_oop->as_instance();
4240     }
4241   }
4242   const Type* con_type = Type::make_constant_from_field(field, holder, field->layout_type(),
4243                                                         /*is_unsigned_load=*/false);
4244   if (con_type != nullptr) {
4245     return makecon(con_type);






4246   }
4247   return nullptr;
4248 }










   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "ci/ciFlatArrayKlass.hpp"
  27 #include "ci/ciInlineKlass.hpp"
  28 #include "ci/ciUtilities.hpp"
  29 #include "classfile/javaClasses.hpp"
  30 #include "ci/ciObjArray.hpp"
  31 #include "asm/register.hpp"
  32 #include "compiler/compileLog.hpp"
  33 #include "gc/shared/barrierSet.hpp"
  34 #include "gc/shared/c2/barrierSetC2.hpp"
  35 #include "interpreter/interpreter.hpp"
  36 #include "memory/resourceArea.hpp"
  37 #include "opto/addnode.hpp"
  38 #include "opto/castnode.hpp"
  39 #include "opto/convertnode.hpp"
  40 #include "opto/graphKit.hpp"
  41 #include "opto/idealKit.hpp"
  42 #include "opto/inlinetypenode.hpp"
  43 #include "opto/intrinsicnode.hpp"
  44 #include "opto/locknode.hpp"
  45 #include "opto/machnode.hpp"
  46 #include "opto/narrowptrnode.hpp"
  47 #include "opto/opaquenode.hpp"
  48 #include "opto/parse.hpp"
  49 #include "opto/rootnode.hpp"
  50 #include "opto/runtime.hpp"
  51 #include "opto/subtypenode.hpp"
  52 #include "runtime/deoptimization.hpp"
  53 #include "runtime/sharedRuntime.hpp"
  54 #include "utilities/bitMap.inline.hpp"
  55 #include "utilities/powerOfTwo.hpp"
  56 #include "utilities/growableArray.hpp"
  57 
  58 //----------------------------GraphKit-----------------------------------------
  59 // Main utility constructor.
  60 GraphKit::GraphKit(JVMState* jvms, PhaseGVN* gvn)
  61   : Phase(Phase::Parser),
  62     _env(C->env()),
  63     _gvn((gvn != nullptr) ? *gvn : *C->initial_gvn()),
  64     _barrier_set(BarrierSet::barrier_set()->barrier_set_c2())
  65 {
  66   assert(gvn == nullptr || !gvn->is_IterGVN() || gvn->is_IterGVN()->delay_transform(), "delay transform should be enabled");
  67   _exceptions = jvms->map()->next_exception();
  68   if (_exceptions != nullptr)  jvms->map()->set_next_exception(nullptr);
  69   set_jvms(jvms);
  70 #ifdef ASSERT
  71   if (_gvn.is_IterGVN() != nullptr) {
  72     assert(_gvn.is_IterGVN()->delay_transform(), "Transformation must be delayed if IterGVN is used");
  73     // Save the initial size of _for_igvn worklist for verification (see ~GraphKit)
  74     _worklist_size = _gvn.C->igvn_worklist()->size();
  75   }
  76 #endif
  77 }
  78 
  79 // Private constructor for parser.
  80 GraphKit::GraphKit()
  81   : Phase(Phase::Parser),
  82     _env(C->env()),
  83     _gvn(*C->initial_gvn()),
  84     _barrier_set(BarrierSet::barrier_set()->barrier_set_c2())
  85 {
  86   _exceptions = nullptr;
  87   set_map(nullptr);
  88   debug_only(_sp = -99);
  89   debug_only(set_bci(-99));
  90 }
  91 
  92 
  93 
  94 //---------------------------clean_stack---------------------------------------
  95 // Clear away rubbish from the stack area of the JVM state.
  96 // This destroys any arguments that may be waiting on the stack.

 852         if (PrintMiscellaneous && (Verbose || WizardMode)) {
 853           tty->print_cr("Zombie local %d: ", local);
 854           jvms->dump();
 855         }
 856         return false;
 857       }
 858     }
 859   }
 860   return true;
 861 }
 862 
 863 #endif //ASSERT
 864 
 865 // Helper function for enforcing certain bytecodes to reexecute if deoptimization happens.
 866 static bool should_reexecute_implied_by_bytecode(JVMState *jvms, bool is_anewarray) {
 867   ciMethod* cur_method = jvms->method();
 868   int       cur_bci   = jvms->bci();
 869   if (cur_method != nullptr && cur_bci != InvocationEntryBci) {
 870     Bytecodes::Code code = cur_method->java_code_at_bci(cur_bci);
 871     return Interpreter::bytecode_should_reexecute(code) ||
 872            (is_anewarray && (code == Bytecodes::_multianewarray));
 873     // Reexecute _multianewarray bytecode which was replaced with
 874     // sequence of [a]newarray. See Parse::do_multianewarray().
 875     //
 876     // Note: interpreter should not have it set since this optimization
 877     // is limited by dimensions and guarded by flag so in some cases
 878     // multianewarray() runtime calls will be generated and
 879     // the bytecode should not be reexecutes (stack will not be reset).
 880   } else {
 881     return false;
 882   }
 883 }
 884 
 885 // Helper function for adding JVMState and debug information to node
 886 void GraphKit::add_safepoint_edges(SafePointNode* call, bool must_throw) {
 887   // Add the safepoint edges to the call (or other safepoint).
 888 
 889   // Make sure dead locals are set to top.  This
 890   // should help register allocation time and cut down on the size
 891   // of the deoptimization information.
 892   assert(dead_locals_are_killed(), "garbage in debug info before safepoint");

1194   Node* conv = _gvn.transform( new ConvI2LNode(offset));
1195   Node* mask = _gvn.transform(ConLNode::make((julong) max_juint));
1196   return _gvn.transform( new AndLNode(conv, mask) );
1197 }
1198 
1199 Node* GraphKit::ConvL2I(Node* offset) {
1200   // short-circuit a common case
1201   jlong offset_con = find_long_con(offset, (jlong)Type::OffsetBot);
1202   if (offset_con != (jlong)Type::OffsetBot) {
1203     return intcon((int) offset_con);
1204   }
1205   return _gvn.transform( new ConvL2INode(offset));
1206 }
1207 
1208 //-------------------------load_object_klass-----------------------------------
1209 Node* GraphKit::load_object_klass(Node* obj) {
1210   // Special-case a fresh allocation to avoid building nodes:
1211   Node* akls = AllocateNode::Ideal_klass(obj, &_gvn);
1212   if (akls != nullptr)  return akls;
1213   Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
1214   return _gvn.transform(LoadKlassNode::make(_gvn, nullptr, immutable_memory(), k_adr, TypeInstPtr::KLASS, TypeInstKlassPtr::OBJECT));
1215 }
1216 
1217 //-------------------------load_array_length-----------------------------------
1218 Node* GraphKit::load_array_length(Node* array) {
1219   // Special-case a fresh allocation to avoid building nodes:
1220   AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(array);
1221   Node *alen;
1222   if (alloc == nullptr) {
1223     Node *r_adr = basic_plus_adr(array, arrayOopDesc::length_offset_in_bytes());
1224     alen = _gvn.transform( new LoadRangeNode(0, immutable_memory(), r_adr, TypeInt::POS));
1225   } else {
1226     alen = array_ideal_length(alloc, _gvn.type(array)->is_oopptr(), false);
1227   }
1228   return alen;
1229 }
1230 
1231 Node* GraphKit::array_ideal_length(AllocateArrayNode* alloc,
1232                                    const TypeOopPtr* oop_type,
1233                                    bool replace_length_in_map) {
1234   Node* length = alloc->Ideal_length();

1243         replace_in_map(length, ccast);
1244       }
1245       return ccast;
1246     }
1247   }
1248   return length;
1249 }
1250 
1251 //------------------------------do_null_check----------------------------------
1252 // Helper function to do a null pointer check.  Returned value is
1253 // the incoming address with null casted away.  You are allowed to use the
1254 // not-null value only if you are control dependent on the test.
1255 #ifndef PRODUCT
1256 extern uint explicit_null_checks_inserted,
1257             explicit_null_checks_elided;
1258 #endif
1259 Node* GraphKit::null_check_common(Node* value, BasicType type,
1260                                   // optional arguments for variations:
1261                                   bool assert_null,
1262                                   Node* *null_control,
1263                                   bool speculative,
1264                                   bool is_init_check) {
1265   assert(!assert_null || null_control == nullptr, "not both at once");
1266   if (stopped())  return top();
1267   NOT_PRODUCT(explicit_null_checks_inserted++);
1268 
1269   if (value->is_InlineType()) {
1270     // Null checking a scalarized but nullable inline type. Check the IsInit
1271     // input instead of the oop input to avoid keeping buffer allocations alive.
1272     InlineTypeNode* vtptr = value->as_InlineType();
1273     while (vtptr->get_oop()->is_InlineType()) {
1274       vtptr = vtptr->get_oop()->as_InlineType();
1275     }
1276     null_check_common(vtptr->get_is_init(), T_INT, assert_null, null_control, speculative, true);
1277     if (stopped()) {
1278       return top();
1279     }
1280     if (assert_null) {
1281       // TODO 8284443 Scalarize here (this currently leads to compilation bailouts)
1282       // vtptr = InlineTypeNode::make_null(_gvn, vtptr->type()->inline_klass());
1283       // replace_in_map(value, vtptr);
1284       // return vtptr;
1285       return null();
1286     }
1287     bool do_replace_in_map = (null_control == nullptr || (*null_control) == top());
1288     return cast_not_null(value, do_replace_in_map);
1289   }
1290 
1291   // Construct null check
1292   Node *chk = nullptr;
1293   switch(type) {
1294     case T_LONG   : chk = new CmpLNode(value, _gvn.zerocon(T_LONG)); break;
1295     case T_INT    : chk = new CmpINode(value, _gvn.intcon(0)); break;
1296     case T_ARRAY  : // fall through
1297       type = T_OBJECT;  // simplify further tests
1298     case T_OBJECT : {
1299       const Type *t = _gvn.type( value );
1300 
1301       const TypeOopPtr* tp = t->isa_oopptr();
1302       if (tp != nullptr && !tp->is_loaded()
1303           // Only for do_null_check, not any of its siblings:
1304           && !assert_null && null_control == nullptr) {
1305         // Usually, any field access or invocation on an unloaded oop type
1306         // will simply fail to link, since the statically linked class is
1307         // likely also to be unloaded.  However, in -Xcomp mode, sometimes
1308         // the static class is loaded but the sharper oop type is not.
1309         // Rather than checking for this obscure case in lots of places,
1310         // we simply observe that a null check on an unloaded class

1374         }
1375         Node *oldcontrol = control();
1376         set_control(cfg);
1377         Node *res = cast_not_null(value);
1378         set_control(oldcontrol);
1379         NOT_PRODUCT(explicit_null_checks_elided++);
1380         return res;
1381       }
1382       cfg = IfNode::up_one_dom(cfg, /*linear_only=*/ true);
1383       if (cfg == nullptr)  break;  // Quit at region nodes
1384       depth++;
1385     }
1386   }
1387 
1388   //-----------
1389   // Branch to failure if null
1390   float ok_prob = PROB_MAX;  // a priori estimate:  nulls never happen
1391   Deoptimization::DeoptReason reason;
1392   if (assert_null) {
1393     reason = Deoptimization::reason_null_assert(speculative);
1394   } else if (type == T_OBJECT || is_init_check) {
1395     reason = Deoptimization::reason_null_check(speculative);
1396   } else {
1397     reason = Deoptimization::Reason_div0_check;
1398   }
1399   // %%% Since Reason_unhandled is not recorded on a per-bytecode basis,
1400   // ciMethodData::has_trap_at will return a conservative -1 if any
1401   // must-be-null assertion has failed.  This could cause performance
1402   // problems for a method after its first do_null_assert failure.
1403   // Consider using 'Reason_class_check' instead?
1404 
1405   // To cause an implicit null check, we set the not-null probability
1406   // to the maximum (PROB_MAX).  For an explicit check the probability
1407   // is set to a smaller value.
1408   if (null_control != nullptr || too_many_traps(reason)) {
1409     // probability is less likely
1410     ok_prob =  PROB_LIKELY_MAG(3);
1411   } else if (!assert_null &&
1412              (ImplicitNullCheckThreshold > 0) &&
1413              method() != nullptr &&
1414              (method()->method_data()->trap_count(reason)

1448   }
1449 
1450   if (assert_null) {
1451     // Cast obj to null on this path.
1452     replace_in_map(value, zerocon(type));
1453     return zerocon(type);
1454   }
1455 
1456   // Cast obj to not-null on this path, if there is no null_control.
1457   // (If there is a null_control, a non-null value may come back to haunt us.)
1458   if (type == T_OBJECT) {
1459     Node* cast = cast_not_null(value, false);
1460     if (null_control == nullptr || (*null_control) == top())
1461       replace_in_map(value, cast);
1462     value = cast;
1463   }
1464 
1465   return value;
1466 }
1467 

1468 //------------------------------cast_not_null----------------------------------
1469 // Cast obj to not-null on this path
1470 Node* GraphKit::cast_not_null(Node* obj, bool do_replace_in_map) {
1471   if (obj->is_InlineType()) {
1472     Node* vt = obj->clone();
1473     vt->as_InlineType()->set_is_init(_gvn);
1474     vt = _gvn.transform(vt);
1475     if (do_replace_in_map) {
1476       replace_in_map(obj, vt);
1477     }
1478     return vt;
1479   }
1480   const Type *t = _gvn.type(obj);
1481   const Type *t_not_null = t->join_speculative(TypePtr::NOTNULL);
1482   // Object is already not-null?
1483   if( t == t_not_null ) return obj;
1484 
1485   Node *cast = new CastPPNode(obj,t_not_null);
1486   cast->init_req(0, control());
1487   cast = _gvn.transform( cast );
1488 
1489   // Scan for instances of 'obj' in the current JVM mapping.
1490   // These instances are known to be not-null after the test.
1491   if (do_replace_in_map)
1492     replace_in_map(obj, cast);
1493 
1494   return cast;                  // Return casted value
1495 }
1496 
1497 // Sometimes in intrinsics, we implicitly know an object is not null
1498 // (there's no actual null check) so we can cast it to not null. In
1499 // the course of optimizations, the input to the cast can become null.

1586 // These are layered on top of the factory methods in LoadNode and StoreNode,
1587 // and integrate with the parser's memory state and _gvn engine.
1588 //
1589 
1590 // factory methods in "int adr_idx"
1591 Node* GraphKit::make_load(Node* ctl, Node* adr, const Type* t, BasicType bt,
1592                           int adr_idx,
1593                           MemNode::MemOrd mo,
1594                           LoadNode::ControlDependency control_dependency,
1595                           bool require_atomic_access,
1596                           bool unaligned,
1597                           bool mismatched,
1598                           bool unsafe,
1599                           uint8_t barrier_data) {
1600   assert(adr_idx != Compile::AliasIdxTop, "use other make_load factory" );
1601   const TypePtr* adr_type = nullptr; // debug-mode-only argument
1602   debug_only(adr_type = C->get_adr_type(adr_idx));
1603   Node* mem = memory(adr_idx);
1604   Node* ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt, mo, control_dependency, require_atomic_access, unaligned, mismatched, unsafe, barrier_data);
1605   ld = _gvn.transform(ld);
1606 
1607   if (((bt == T_OBJECT) && C->do_escape_analysis()) || C->eliminate_boxing()) {
1608     // Improve graph before escape analysis and boxing elimination.
1609     record_for_igvn(ld);
1610   }
1611   return ld;
1612 }
1613 
1614 Node* GraphKit::store_to_memory(Node* ctl, Node* adr, Node *val, BasicType bt,
1615                                 int adr_idx,
1616                                 MemNode::MemOrd mo,
1617                                 bool require_atomic_access,
1618                                 bool unaligned,
1619                                 bool mismatched,
1620                                 bool unsafe,
1621                                 int barrier_data) {
1622   assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
1623   const TypePtr* adr_type = nullptr;
1624   debug_only(adr_type = C->get_adr_type(adr_idx));
1625   Node *mem = memory(adr_idx);
1626   Node* st = StoreNode::make(_gvn, ctl, mem, adr, adr_type, val, bt, mo, require_atomic_access);

1633   if (unsafe) {
1634     st->as_Store()->set_unsafe_access();
1635   }
1636   st->as_Store()->set_barrier_data(barrier_data);
1637   st = _gvn.transform(st);
1638   set_memory(st, adr_idx);
1639   // Back-to-back stores can only remove intermediate store with DU info
1640   // so push on worklist for optimizer.
1641   if (mem->req() > MemNode::Address && adr == mem->in(MemNode::Address))
1642     record_for_igvn(st);
1643 
1644   return st;
1645 }
1646 
1647 Node* GraphKit::access_store_at(Node* obj,
1648                                 Node* adr,
1649                                 const TypePtr* adr_type,
1650                                 Node* val,
1651                                 const Type* val_type,
1652                                 BasicType bt,
1653                                 DecoratorSet decorators,
1654                                 bool safe_for_replace) {
1655   // Transformation of a value which could be null pointer (CastPP #null)
1656   // could be delayed during Parse (for example, in adjust_map_after_if()).
1657   // Execute transformation here to avoid barrier generation in such case.
1658   if (_gvn.type(val) == TypePtr::NULL_PTR) {
1659     val = _gvn.makecon(TypePtr::NULL_PTR);
1660   }
1661 
1662   if (stopped()) {
1663     return top(); // Dead path ?
1664   }
1665 
1666   assert(val != nullptr, "not dead path");
1667   if (val->is_InlineType()) {
1668     // Store to non-flat field. Buffer the inline type and make sure
1669     // the store is re-executed if the allocation triggers deoptimization.
1670     PreserveReexecuteState preexecs(this);
1671     jvms()->set_should_reexecute(true);
1672     val = val->as_InlineType()->buffer(this, safe_for_replace);
1673   }
1674 
1675   C2AccessValuePtr addr(adr, adr_type);
1676   C2AccessValue value(val, val_type);
1677   C2ParseAccess access(this, decorators | C2_WRITE_ACCESS, bt, obj, addr);
1678   if (access.is_raw()) {
1679     return _barrier_set->BarrierSetC2::store_at(access, value);
1680   } else {
1681     return _barrier_set->store_at(access, value);
1682   }
1683 }
1684 
1685 Node* GraphKit::access_load_at(Node* obj,   // containing obj
1686                                Node* adr,   // actual address to store val at
1687                                const TypePtr* adr_type,
1688                                const Type* val_type,
1689                                BasicType bt,
1690                                DecoratorSet decorators,
1691                                Node* ctl) {
1692   if (stopped()) {
1693     return top(); // Dead path ?
1694   }
1695 
1696   C2AccessValuePtr addr(adr, adr_type);
1697   C2ParseAccess access(this, decorators | C2_READ_ACCESS, bt, obj, addr, ctl);
1698   if (access.is_raw()) {
1699     return _barrier_set->BarrierSetC2::load_at(access, val_type);
1700   } else {
1701     return _barrier_set->load_at(access, val_type);
1702   }
1703 }
1704 
1705 Node* GraphKit::access_load(Node* adr,   // actual address to load val at
1706                             const Type* val_type,
1707                             BasicType bt,
1708                             DecoratorSet decorators) {
1709   if (stopped()) {
1710     return top(); // Dead path ?
1711   }
1712 
1713   C2AccessValuePtr addr(adr, adr->bottom_type()->is_ptr());
1714   C2ParseAccess access(this, decorators | C2_READ_ACCESS, bt, nullptr, addr);
1715   if (access.is_raw()) {
1716     return _barrier_set->BarrierSetC2::load_at(access, val_type);
1717   } else {

1782                                      Node* new_val,
1783                                      const Type* value_type,
1784                                      BasicType bt,
1785                                      DecoratorSet decorators) {
1786   C2AccessValuePtr addr(adr, adr_type);
1787   C2AtomicParseAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, bt, obj, addr, alias_idx);
1788   if (access.is_raw()) {
1789     return _barrier_set->BarrierSetC2::atomic_add_at(access, new_val, value_type);
1790   } else {
1791     return _barrier_set->atomic_add_at(access, new_val, value_type);
1792   }
1793 }
1794 
1795 void GraphKit::access_clone(Node* src, Node* dst, Node* size, bool is_array) {
1796   return _barrier_set->clone(this, src, dst, size, is_array);
1797 }
1798 
1799 //-------------------------array_element_address-------------------------
1800 Node* GraphKit::array_element_address(Node* ary, Node* idx, BasicType elembt,
1801                                       const TypeInt* sizetype, Node* ctrl) {
1802   const TypeAryPtr* arytype = _gvn.type(ary)->is_aryptr();
1803   uint shift = arytype->is_flat() ? arytype->flat_log_elem_size() : exact_log2(type2aelembytes(elembt));
1804   uint header = arrayOopDesc::base_offset_in_bytes(elembt);
1805 
1806   // short-circuit a common case (saves lots of confusing waste motion)
1807   jint idx_con = find_int_con(idx, -1);
1808   if (idx_con >= 0) {
1809     intptr_t offset = header + ((intptr_t)idx_con << shift);
1810     return basic_plus_adr(ary, offset);
1811   }
1812 
1813   // must be correct type for alignment purposes
1814   Node* base  = basic_plus_adr(ary, header);
1815   idx = Compile::conv_I2X_index(&_gvn, idx, sizetype, ctrl);
1816   Node* scale = _gvn.transform( new LShiftXNode(idx, intcon(shift)) );
1817   return basic_plus_adr(ary, base, scale);
1818 }
1819 
1820 //-------------------------load_array_element-------------------------
1821 Node* GraphKit::load_array_element(Node* ary, Node* idx, const TypeAryPtr* arytype, bool set_ctrl) {
1822   const Type* elemtype = arytype->elem();
1823   BasicType elembt = elemtype->array_element_basic_type();
1824   Node* adr = array_element_address(ary, idx, elembt, arytype->size());
1825   if (elembt == T_NARROWOOP) {
1826     elembt = T_OBJECT; // To satisfy switch in LoadNode::make()
1827   }
1828   Node* ld = access_load_at(ary, adr, arytype, elemtype, elembt,
1829                             IN_HEAP | IS_ARRAY | (set_ctrl ? C2_CONTROL_DEPENDENT_LOAD : 0));
1830   return ld;
1831 }
1832 
1833 //-------------------------set_arguments_for_java_call-------------------------
1834 // Arguments (pre-popped from the stack) are taken from the JVMS.
1835 void GraphKit::set_arguments_for_java_call(CallJavaNode* call, bool is_late_inline) {
1836   PreserveReexecuteState preexecs(this);
1837   if (EnableValhalla) {
1838     // Make sure the call is "re-executed", if buffering of inline type arguments triggers deoptimization.
1839     // At this point, the call hasn't been executed yet, so we will only ever execute the call once.
1840     jvms()->set_should_reexecute(true);
1841     int arg_size = method()->get_declared_signature_at_bci(bci())->arg_size_for_bc(java_bc());
1842     inc_sp(arg_size);
1843   }
1844   // Add the call arguments
1845   const TypeTuple* domain = call->tf()->domain_sig();
1846   uint nargs = domain->cnt();
1847   int arg_num = 0;
1848   for (uint i = TypeFunc::Parms, idx = TypeFunc::Parms; i < nargs; i++) {
1849     Node* arg = argument(i-TypeFunc::Parms);
1850     const Type* t = domain->field_at(i);
1851     // TODO 8284443 A static call to a mismatched method should still be scalarized
1852     if (t->is_inlinetypeptr() && !call->method()->get_Method()->mismatch() && call->method()->is_scalarized_arg(arg_num)) {
1853       // We don't pass inline type arguments by reference but instead pass each field of the inline type
1854       if (!arg->is_InlineType()) {
1855         assert(_gvn.type(arg)->is_zero_type() && !t->inline_klass()->is_null_free(), "Unexpected argument type");
1856         arg = InlineTypeNode::make_from_oop(this, arg, t->inline_klass(), t->inline_klass()->is_null_free());
1857       }
1858       InlineTypeNode* vt = arg->as_InlineType();
1859       vt->pass_fields(this, call, idx, true, !t->maybe_null());
1860       // If an inline type argument is passed as fields, attach the Method* to the call site
1861       // to be able to access the extended signature later via attached_method_before_pc().
1862       // For example, see CompiledMethod::preserve_callee_argument_oops().
1863       call->set_override_symbolic_info(true);
1864       // Register an evol dependency on the callee method to make sure that this method is deoptimized and
1865       // re-compiled with a non-scalarized calling convention if the callee method is later marked as mismatched.
1866       C->dependencies()->assert_evol_method(call->method());
1867       arg_num++;
1868       continue;
1869     } else if (arg->is_InlineType()) {
1870       // Pass inline type argument via oop to callee
1871       arg = arg->as_InlineType()->buffer(this);
1872       if (!is_late_inline) {
1873         arg = arg->as_InlineType()->get_oop();
1874       }
1875     }
1876     if (t != Type::HALF) {
1877       arg_num++;
1878     }
1879     call->init_req(idx++, arg);
1880   }
1881 }
1882 
1883 //---------------------------set_edges_for_java_call---------------------------
1884 // Connect a newly created call into the current JVMS.
1885 // A return value node (if any) is returned from set_edges_for_java_call.
1886 void GraphKit::set_edges_for_java_call(CallJavaNode* call, bool must_throw, bool separate_io_proj) {
1887 
1888   // Add the predefined inputs:
1889   call->init_req( TypeFunc::Control, control() );
1890   call->init_req( TypeFunc::I_O    , i_o() );
1891   call->init_req( TypeFunc::Memory , reset_memory() );
1892   call->init_req( TypeFunc::FramePtr, frameptr() );
1893   call->init_req( TypeFunc::ReturnAdr, top() );
1894 
1895   add_safepoint_edges(call, must_throw);
1896 
1897   Node* xcall = _gvn.transform(call);
1898 
1899   if (xcall == top()) {
1900     set_control(top());
1901     return;
1902   }
1903   assert(xcall == call, "call identity is stable");
1904 
1905   // Re-use the current map to produce the result.
1906 
1907   set_control(_gvn.transform(new ProjNode(call, TypeFunc::Control)));
1908   set_i_o(    _gvn.transform(new ProjNode(call, TypeFunc::I_O    , separate_io_proj)));
1909   set_all_memory_call(xcall, separate_io_proj);
1910 
1911   //return xcall;   // no need, caller already has it
1912 }
1913 
1914 Node* GraphKit::set_results_for_java_call(CallJavaNode* call, bool separate_io_proj, bool deoptimize) {
1915   if (stopped())  return top();  // maybe the call folded up?
1916 







1917   // Note:  Since any out-of-line call can produce an exception,
1918   // we always insert an I_O projection from the call into the result.
1919 
1920   make_slow_call_ex(call, env()->Throwable_klass(), separate_io_proj, deoptimize);
1921 
1922   if (separate_io_proj) {
1923     // The caller requested separate projections be used by the fall
1924     // through and exceptional paths, so replace the projections for
1925     // the fall through path.
1926     set_i_o(_gvn.transform( new ProjNode(call, TypeFunc::I_O) ));
1927     set_all_memory(_gvn.transform( new ProjNode(call, TypeFunc::Memory) ));
1928   }
1929 
1930   // Capture the return value, if any.
1931   Node* ret;
1932   if (call->method() == nullptr || call->method()->return_type()->basic_type() == T_VOID) {
1933     ret = top();
1934   } else if (call->tf()->returns_inline_type_as_fields()) {
1935     // Return of multiple values (inline type fields): we create a
1936     // InlineType node, each field is a projection from the call.
1937     ciInlineKlass* vk = call->method()->return_type()->as_inline_klass();
1938     uint base_input = TypeFunc::Parms;
1939     // ret = InlineTypeNode::make_from_multi(this, call, vk, base_input, false, call->method()->signature()->returns_null_free_inline_type());
1940     ret = InlineTypeNode::make_from_multi(this, call, vk, base_input, false, false); // JDK-8325660: revisit this code after removal of Q-descriptors
1941   } else {
1942     ret = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
1943     ciType* t = call->method()->return_type();
1944     if (t->is_klass()) {
1945       const Type* type = TypeOopPtr::make_from_klass(t->as_klass());
1946       if (type->is_inlinetypeptr()) {
1947         ret = InlineTypeNode::make_from_oop(this, ret, type->inline_klass(), type->inline_klass()->is_null_free());
1948       }
1949     }
1950   }
1951 
1952   return ret;
1953 }
1954 
1955 //--------------------set_predefined_input_for_runtime_call--------------------
1956 // Reading and setting the memory state is way conservative here.
1957 // The real problem is that I am not doing real Type analysis on memory,
1958 // so I cannot distinguish card mark stores from other stores.  Across a GC
1959 // point the Store Barrier and the card mark memory has to agree.  I cannot
1960 // have a card mark store and its barrier split across the GC point from
1961 // either above or below.  Here I get that to happen by reading ALL of memory.
1962 // A better answer would be to separate out card marks from other memory.
1963 // For now, return the input memory state, so that it can be reused
1964 // after the call, if this call has restricted memory effects.
1965 Node* GraphKit::set_predefined_input_for_runtime_call(SafePointNode* call, Node* narrow_mem) {
1966   // Set fixed predefined input arguments
1967   Node* memory = reset_memory();
1968   Node* m = narrow_mem == nullptr ? memory : narrow_mem;
1969   call->init_req( TypeFunc::Control,   control()  );
1970   call->init_req( TypeFunc::I_O,       top()      ); // does no i/o
1971   call->init_req( TypeFunc::Memory,    m          ); // may gc ptrs

2022     if (use->is_MergeMem()) {
2023       wl.push(use);
2024     }
2025   }
2026 }
2027 
2028 // Replace the call with the current state of the kit.
2029 void GraphKit::replace_call(CallNode* call, Node* result, bool do_replaced_nodes) {
2030   JVMState* ejvms = nullptr;
2031   if (has_exceptions()) {
2032     ejvms = transfer_exceptions_into_jvms();
2033   }
2034 
2035   ReplacedNodes replaced_nodes = map()->replaced_nodes();
2036   ReplacedNodes replaced_nodes_exception;
2037   Node* ex_ctl = top();
2038 
2039   SafePointNode* final_state = stop();
2040 
2041   // Find all the needed outputs of this call
2042   CallProjections* callprojs = call->extract_projections(true);

2043 
2044   Unique_Node_List wl;
2045   Node* init_mem = call->in(TypeFunc::Memory);
2046   Node* final_mem = final_state->in(TypeFunc::Memory);
2047   Node* final_ctl = final_state->in(TypeFunc::Control);
2048   Node* final_io = final_state->in(TypeFunc::I_O);
2049 
2050   // Replace all the old call edges with the edges from the inlining result
2051   if (callprojs->fallthrough_catchproj != nullptr) {
2052     C->gvn_replace_by(callprojs->fallthrough_catchproj, final_ctl);
2053   }
2054   if (callprojs->fallthrough_memproj != nullptr) {
2055     if (final_mem->is_MergeMem()) {
2056       // Parser's exits MergeMem was not transformed but may be optimized
2057       final_mem = _gvn.transform(final_mem);
2058     }
2059     C->gvn_replace_by(callprojs->fallthrough_memproj,   final_mem);
2060     add_mergemem_users_to_worklist(wl, final_mem);
2061   }
2062   if (callprojs->fallthrough_ioproj != nullptr) {
2063     C->gvn_replace_by(callprojs->fallthrough_ioproj,    final_io);
2064   }
2065 
2066   // Replace the result with the new result if it exists and is used
2067   if (callprojs->resproj[0] != nullptr && result != nullptr) {
2068     // If the inlined code is dead, the result projections for an inline type returned as
2069     // fields have not been replaced. They will go away once the call is replaced by TOP below.
2070     assert(callprojs->nb_resproj == 1 || (call->tf()->returns_inline_type_as_fields() && stopped()),
2071            "unexpected number of results");
2072     C->gvn_replace_by(callprojs->resproj[0], result);
2073   }
2074 
2075   if (ejvms == nullptr) {
2076     // No exception edges to simply kill off those paths
2077     if (callprojs->catchall_catchproj != nullptr) {
2078       C->gvn_replace_by(callprojs->catchall_catchproj, C->top());
2079     }
2080     if (callprojs->catchall_memproj != nullptr) {
2081       C->gvn_replace_by(callprojs->catchall_memproj,   C->top());
2082     }
2083     if (callprojs->catchall_ioproj != nullptr) {
2084       C->gvn_replace_by(callprojs->catchall_ioproj,    C->top());
2085     }
2086     // Replace the old exception object with top
2087     if (callprojs->exobj != nullptr) {
2088       C->gvn_replace_by(callprojs->exobj, C->top());
2089     }
2090   } else {
2091     GraphKit ekit(ejvms);
2092 
2093     // Load my combined exception state into the kit, with all phis transformed:
2094     SafePointNode* ex_map = ekit.combine_and_pop_all_exception_states();
2095     replaced_nodes_exception = ex_map->replaced_nodes();
2096 
2097     Node* ex_oop = ekit.use_exception_state(ex_map);
2098 
2099     if (callprojs->catchall_catchproj != nullptr) {
2100       C->gvn_replace_by(callprojs->catchall_catchproj, ekit.control());
2101       ex_ctl = ekit.control();
2102     }
2103     if (callprojs->catchall_memproj != nullptr) {
2104       Node* ex_mem = ekit.reset_memory();
2105       C->gvn_replace_by(callprojs->catchall_memproj,   ex_mem);
2106       add_mergemem_users_to_worklist(wl, ex_mem);
2107     }
2108     if (callprojs->catchall_ioproj != nullptr) {
2109       C->gvn_replace_by(callprojs->catchall_ioproj,    ekit.i_o());
2110     }
2111 
2112     // Replace the old exception object with the newly created one
2113     if (callprojs->exobj != nullptr) {
2114       C->gvn_replace_by(callprojs->exobj, ex_oop);
2115     }
2116   }
2117 
2118   // Disconnect the call from the graph
2119   call->disconnect_inputs(C);
2120   C->gvn_replace_by(call, C->top());
2121 
2122   // Clean up any MergeMems that feed other MergeMems since the
2123   // optimizer doesn't like that.
2124   while (wl.size() > 0) {
2125     _gvn.transform(wl.pop());
2126   }
2127 
2128   if (callprojs->fallthrough_catchproj != nullptr && !final_ctl->is_top() && do_replaced_nodes) {
2129     replaced_nodes.apply(C, final_ctl);
2130   }
2131   if (!ex_ctl->is_top() && do_replaced_nodes) {
2132     replaced_nodes_exception.apply(C, ex_ctl);
2133   }
2134 }
2135 
2136 
2137 //------------------------------increment_counter------------------------------
2138 // for statistics: increment a VM counter by 1
2139 
2140 void GraphKit::increment_counter(address counter_addr) {
2141   Node* adr1 = makecon(TypeRawPtr::make(counter_addr));
2142   increment_counter(adr1);
2143 }
2144 
2145 void GraphKit::increment_counter(Node* counter_addr) {
2146   int adr_type = Compile::AliasIdxRaw;
2147   Node* ctrl = control();
2148   Node* cnt  = make_load(ctrl, counter_addr, TypeLong::LONG, T_LONG, adr_type, MemNode::unordered);

2307  *
2308  * @param n          node that the type applies to
2309  * @param exact_kls  type from profiling
2310  * @param maybe_null did profiling see null?
2311  *
2312  * @return           node with improved type
2313  */
2314 Node* GraphKit::record_profile_for_speculation(Node* n, ciKlass* exact_kls, ProfilePtrKind ptr_kind) {
2315   const Type* current_type = _gvn.type(n);
2316   assert(UseTypeSpeculation, "type speculation must be on");
2317 
2318   const TypePtr* speculative = current_type->speculative();
2319 
2320   // Should the klass from the profile be recorded in the speculative type?
2321   if (current_type->would_improve_type(exact_kls, jvms()->depth())) {
2322     const TypeKlassPtr* tklass = TypeKlassPtr::make(exact_kls, Type::trust_interfaces);
2323     const TypeOopPtr* xtype = tklass->as_instance_type();
2324     assert(xtype->klass_is_exact(), "Should be exact");
2325     // Any reason to believe n is not null (from this profiling or a previous one)?
2326     assert(ptr_kind != ProfileAlwaysNull, "impossible here");
2327     const TypePtr* ptr = (ptr_kind != ProfileNeverNull && current_type->speculative_maybe_null()) ? TypePtr::BOTTOM : TypePtr::NOTNULL;
2328     // record the new speculative type's depth
2329     speculative = xtype->cast_to_ptr_type(ptr->ptr())->is_ptr();
2330     speculative = speculative->with_inline_depth(jvms()->depth());
2331   } else if (current_type->would_improve_ptr(ptr_kind)) {
2332     // Profiling report that null was never seen so we can change the
2333     // speculative type to non null ptr.
2334     if (ptr_kind == ProfileAlwaysNull) {
2335       speculative = TypePtr::NULL_PTR;
2336     } else {
2337       assert(ptr_kind == ProfileNeverNull, "nothing else is an improvement");
2338       const TypePtr* ptr = TypePtr::NOTNULL;
2339       if (speculative != nullptr) {
2340         speculative = speculative->cast_to_ptr_type(ptr->ptr())->is_ptr();
2341       } else {
2342         speculative = ptr;
2343       }
2344     }
2345   }
2346 
2347   if (speculative != current_type->speculative()) {
2348     // Build a type with a speculative type (what we think we know
2349     // about the type but will need a guard when we use it)
2350     const TypeOopPtr* spec_type = TypeOopPtr::make(TypePtr::BotPTR, Type::Offset::bottom, TypeOopPtr::InstanceBot, speculative);
2351     // We're changing the type, we need a new CheckCast node to carry
2352     // the new type. The new type depends on the control: what
2353     // profiling tells us is only valid from here as far as we can
2354     // tell.
2355     Node* cast = new CheckCastPPNode(control(), n, current_type->remove_speculative()->join_speculative(spec_type));
2356     cast = _gvn.transform(cast);
2357     replace_in_map(n, cast);
2358     n = cast;
2359   }
2360 
2361   return n;
2362 }
2363 
2364 /**
2365  * Record profiling data from receiver profiling at an invoke with the
2366  * type system so that it can propagate it (speculation)
2367  *
2368  * @param n  receiver node
2369  *
2370  * @return   node with improved type
2371  */
2372 Node* GraphKit::record_profiled_receiver_for_speculation(Node* n) {
2373   if (!UseTypeSpeculation) {
2374     return n;
2375   }
2376   ciKlass* exact_kls = profile_has_unique_klass();
2377   ProfilePtrKind ptr_kind = ProfileMaybeNull;
2378   if ((java_bc() == Bytecodes::_checkcast ||
2379        java_bc() == Bytecodes::_instanceof ||
2380        java_bc() == Bytecodes::_aastore) &&
2381       method()->method_data()->is_mature()) {
2382     ciProfileData* data = method()->method_data()->bci_to_data(bci());
2383     if (data != nullptr) {
2384       if (java_bc() == Bytecodes::_aastore) {
2385         ciKlass* array_type = nullptr;
2386         ciKlass* element_type = nullptr;
2387         ProfilePtrKind element_ptr = ProfileMaybeNull;
2388         bool flat_array = true;
2389         bool null_free_array = true;
2390         method()->array_access_profiled_type(bci(), array_type, element_type, element_ptr, flat_array, null_free_array);
2391         exact_kls = element_type;
2392         ptr_kind = element_ptr;
2393       } else {
2394         if (!data->as_BitData()->null_seen()) {
2395           ptr_kind = ProfileNeverNull;
2396         } else {
2397           assert(data->is_ReceiverTypeData(), "bad profile data type");
2398           ciReceiverTypeData* call = (ciReceiverTypeData*)data->as_ReceiverTypeData();
2399           uint i = 0;
2400           for (; i < call->row_limit(); i++) {
2401             ciKlass* receiver = call->receiver(i);
2402             if (receiver != nullptr) {
2403               break;
2404             }
2405           }
2406           ptr_kind = (i == call->row_limit()) ? ProfileAlwaysNull : ProfileMaybeNull;
2407         }

2408       }
2409     }
2410   }
2411   return record_profile_for_speculation(n, exact_kls, ptr_kind);
2412 }
2413 
2414 /**
2415  * Record profiling data from argument profiling at an invoke with the
2416  * type system so that it can propagate it (speculation)
2417  *
2418  * @param dest_method  target method for the call
2419  * @param bc           what invoke bytecode is this?
2420  */
2421 void GraphKit::record_profiled_arguments_for_speculation(ciMethod* dest_method, Bytecodes::Code bc) {
2422   if (!UseTypeSpeculation) {
2423     return;
2424   }
2425   const TypeFunc* tf    = TypeFunc::make(dest_method);
2426   int             nargs = tf->domain_sig()->cnt() - TypeFunc::Parms;
2427   int skip = Bytecodes::has_receiver(bc) ? 1 : 0;
2428   for (int j = skip, i = 0; j < nargs && i < TypeProfileArgsLimit; j++) {
2429     const Type *targ = tf->domain_sig()->field_at(j + TypeFunc::Parms);
2430     if (is_reference_type(targ->basic_type())) {
2431       ProfilePtrKind ptr_kind = ProfileMaybeNull;
2432       ciKlass* better_type = nullptr;
2433       if (method()->argument_profiled_type(bci(), i, better_type, ptr_kind)) {
2434         record_profile_for_speculation(argument(j), better_type, ptr_kind);
2435       }
2436       i++;
2437     }
2438   }
2439 }
2440 
2441 /**
2442  * Record profiling data from parameter profiling at an invoke with
2443  * the type system so that it can propagate it (speculation)
2444  */
2445 void GraphKit::record_profiled_parameters_for_speculation() {
2446   if (!UseTypeSpeculation) {
2447     return;
2448   }
2449   for (int i = 0, j = 0; i < method()->arg_size() ; i++) {

2463  * the type system so that it can propagate it (speculation)
2464  */
2465 void GraphKit::record_profiled_return_for_speculation() {
2466   if (!UseTypeSpeculation) {
2467     return;
2468   }
2469   ProfilePtrKind ptr_kind = ProfileMaybeNull;
2470   ciKlass* better_type = nullptr;
2471   if (method()->return_profiled_type(bci(), better_type, ptr_kind)) {
2472     // If profiling reports a single type for the return value,
2473     // feed it to the type system so it can propagate it as a
2474     // speculative type
2475     record_profile_for_speculation(stack(sp()-1), better_type, ptr_kind);
2476   }
2477 }
2478 
2479 void GraphKit::round_double_arguments(ciMethod* dest_method) {
2480   if (Matcher::strict_fp_requires_explicit_rounding) {
2481     // (Note:  TypeFunc::make has a cache that makes this fast.)
2482     const TypeFunc* tf    = TypeFunc::make(dest_method);
2483     int             nargs = tf->domain_sig()->cnt() - TypeFunc::Parms;
2484     for (int j = 0; j < nargs; j++) {
2485       const Type *targ = tf->domain_sig()->field_at(j + TypeFunc::Parms);
2486       if (targ->basic_type() == T_DOUBLE) {
2487         // If any parameters are doubles, they must be rounded before
2488         // the call, dprecision_rounding does gvn.transform
2489         Node *arg = argument(j);
2490         arg = dprecision_rounding(arg);
2491         set_argument(j, arg);
2492       }
2493     }
2494   }
2495 }
2496 
2497 // rounding for strict float precision conformance
2498 Node* GraphKit::precision_rounding(Node* n) {
2499   if (Matcher::strict_fp_requires_explicit_rounding) {
2500 #ifdef IA32
2501     if (UseSSE == 0) {
2502       return _gvn.transform(new RoundFloatNode(0, n));
2503     }
2504 #else
2505     Unimplemented();

2614                                   // The first null ends the list.
2615                                   Node* parm0, Node* parm1,
2616                                   Node* parm2, Node* parm3,
2617                                   Node* parm4, Node* parm5,
2618                                   Node* parm6, Node* parm7) {
2619   assert(call_addr != nullptr, "must not call null targets");
2620 
2621   // Slow-path call
2622   bool is_leaf = !(flags & RC_NO_LEAF);
2623   bool has_io  = (!is_leaf && !(flags & RC_NO_IO));
2624   if (call_name == nullptr) {
2625     assert(!is_leaf, "must supply name for leaf");
2626     call_name = OptoRuntime::stub_name(call_addr);
2627   }
2628   CallNode* call;
2629   if (!is_leaf) {
2630     call = new CallStaticJavaNode(call_type, call_addr, call_name, adr_type);
2631   } else if (flags & RC_NO_FP) {
2632     call = new CallLeafNoFPNode(call_type, call_addr, call_name, adr_type);
2633   } else  if (flags & RC_VECTOR){
2634     uint num_bits = call_type->range_sig()->field_at(TypeFunc::Parms)->is_vect()->length_in_bytes() * BitsPerByte;
2635     call = new CallLeafVectorNode(call_type, call_addr, call_name, adr_type, num_bits);
2636   } else {
2637     call = new CallLeafNode(call_type, call_addr, call_name, adr_type);
2638   }
2639 
2640   // The following is similar to set_edges_for_java_call,
2641   // except that the memory effects of the call are restricted to AliasIdxRaw.
2642 
2643   // Slow path call has no side-effects, uses few values
2644   bool wide_in  = !(flags & RC_NARROW_MEM);
2645   bool wide_out = (C->get_alias_index(adr_type) == Compile::AliasIdxBot);
2646 
2647   Node* prev_mem = nullptr;
2648   if (wide_in) {
2649     prev_mem = set_predefined_input_for_runtime_call(call);
2650   } else {
2651     assert(!wide_out, "narrow in => narrow out");
2652     Node* narrow_mem = memory(adr_type);
2653     prev_mem = set_predefined_input_for_runtime_call(call, narrow_mem);
2654   }

2694 
2695   if (has_io) {
2696     set_i_o(_gvn.transform(new ProjNode(call, TypeFunc::I_O)));
2697   }
2698   return call;
2699 
2700 }
2701 
2702 // i2b
2703 Node* GraphKit::sign_extend_byte(Node* in) {
2704   Node* tmp = _gvn.transform(new LShiftINode(in, _gvn.intcon(24)));
2705   return _gvn.transform(new RShiftINode(tmp, _gvn.intcon(24)));
2706 }
2707 
2708 // i2s
2709 Node* GraphKit::sign_extend_short(Node* in) {
2710   Node* tmp = _gvn.transform(new LShiftINode(in, _gvn.intcon(16)));
2711   return _gvn.transform(new RShiftINode(tmp, _gvn.intcon(16)));
2712 }
2713 
2714 
2715 //------------------------------merge_memory-----------------------------------
2716 // Merge memory from one path into the current memory state.
2717 void GraphKit::merge_memory(Node* new_mem, Node* region, int new_path) {
2718   for (MergeMemStream mms(merged_memory(), new_mem->as_MergeMem()); mms.next_non_empty2(); ) {
2719     Node* old_slice = mms.force_memory();
2720     Node* new_slice = mms.memory2();
2721     if (old_slice != new_slice) {
2722       PhiNode* phi;
2723       if (old_slice->is_Phi() && old_slice->as_Phi()->region() == region) {
2724         if (mms.is_empty()) {
2725           // clone base memory Phi's inputs for this memory slice
2726           assert(old_slice == mms.base_memory(), "sanity");
2727           phi = PhiNode::make(region, nullptr, Type::MEMORY, mms.adr_type(C));
2728           _gvn.set_type(phi, Type::MEMORY);
2729           for (uint i = 1; i < phi->req(); i++) {
2730             phi->init_req(i, old_slice->in(i));
2731           }
2732         } else {
2733           phi = old_slice->as_Phi(); // Phi was generated already
2734         }

2991 
2992   // Now do a linear scan of the secondary super-klass array.  Again, no real
2993   // performance impact (too rare) but it's gotta be done.
2994   // Since the code is rarely used, there is no penalty for moving it
2995   // out of line, and it can only improve I-cache density.
2996   // The decision to inline or out-of-line this final check is platform
2997   // dependent, and is found in the AD file definition of PartialSubtypeCheck.
2998   Node* psc = gvn.transform(
2999     new PartialSubtypeCheckNode(*ctrl, subklass, superklass));
3000 
3001   IfNode *iff4 = gen_subtype_check_compare(*ctrl, psc, gvn.zerocon(T_OBJECT), BoolTest::ne, PROB_FAIR, gvn, T_ADDRESS);
3002   r_not_subtype->init_req(2, gvn.transform(new IfTrueNode (iff4)));
3003   r_ok_subtype ->init_req(3, gvn.transform(new IfFalseNode(iff4)));
3004 
3005   // Return false path; set default control to true path.
3006   *ctrl = gvn.transform(r_ok_subtype);
3007   return gvn.transform(r_not_subtype);
3008 }
3009 
3010 Node* GraphKit::gen_subtype_check(Node* obj_or_subklass, Node* superklass) {
3011   const Type* sub_t = _gvn.type(obj_or_subklass);
3012   if (sub_t->make_oopptr() != nullptr && sub_t->make_oopptr()->is_inlinetypeptr()) {
3013     sub_t = TypeKlassPtr::make(sub_t->inline_klass());
3014     obj_or_subklass = makecon(sub_t);
3015   }
3016   bool expand_subtype_check = C->post_loop_opts_phase() ||   // macro node expansion is over
3017                               ExpandSubTypeCheckAtParseTime; // forced expansion
3018   if (expand_subtype_check) {
3019     MergeMemNode* mem = merged_memory();
3020     Node* ctrl = control();
3021     Node* subklass = obj_or_subklass;
3022     if (!sub_t->isa_klassptr()) {
3023       subklass = load_object_klass(obj_or_subklass);
3024     }
3025 
3026     Node* n = Phase::gen_subtype_check(subklass, superklass, &ctrl, mem, _gvn, method(), bci());
3027     set_control(ctrl);
3028     return n;
3029   }
3030 
3031   Node* check = _gvn.transform(new SubTypeCheckNode(C, obj_or_subklass, superklass, method(), bci()));
3032   Node* bol = _gvn.transform(new BoolNode(check, BoolTest::eq));
3033   IfNode* iff = create_and_xform_if(control(), bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
3034   set_control(_gvn.transform(new IfTrueNode(iff)));
3035   return _gvn.transform(new IfFalseNode(iff));
3036 }
3037 
3038 // Profile-driven exact type check:
3039 Node* GraphKit::type_check_receiver(Node* receiver, ciKlass* klass,
3040                                     float prob, Node* *casted_receiver) {

3041   assert(!klass->is_interface(), "no exact type check on interfaces");
3042   Node* fail = top();
3043   const Type* rec_t = _gvn.type(receiver);
3044   if (rec_t->is_inlinetypeptr()) {
3045     if (klass->equals(rec_t->inline_klass())) {
3046       (*casted_receiver) = receiver; // Always passes
3047     } else {
3048       (*casted_receiver) = top();    // Always fails
3049       fail = control();
3050       set_control(top());
3051     }
3052     return fail;
3053   }
3054   const TypeKlassPtr* tklass = TypeKlassPtr::make(klass, Type::trust_interfaces);
3055   Node* recv_klass = load_object_klass(receiver);
3056   fail = type_check(recv_klass, tklass, prob);





3057 
3058   if (!stopped()) {
3059     const TypeOopPtr* receiver_type = _gvn.type(receiver)->isa_oopptr();
3060     const TypeOopPtr* recv_xtype = tklass->as_instance_type();
3061     assert(recv_xtype->klass_is_exact(), "");
3062 
3063     if (!receiver_type->higher_equal(recv_xtype)) { // ignore redundant casts
3064       // Subsume downstream occurrences of receiver with a cast to
3065       // recv_xtype, since now we know what the type will be.
3066       Node* cast = new CheckCastPPNode(control(), receiver, recv_xtype);
3067       Node* res = _gvn.transform(cast);
3068       if (recv_xtype->is_inlinetypeptr()) {
3069         assert(!gvn().type(res)->maybe_null(), "receiver should never be null");
3070         res = InlineTypeNode::make_from_oop(this, res, recv_xtype->inline_klass());
3071       }
3072       (*casted_receiver) = res;
3073       assert(!(*casted_receiver)->is_top(), "that path should be unreachable");
3074       // (User must make the replace_in_map call.)
3075     }
3076   }
3077 
3078   return fail;
3079 }
3080 
3081 Node* GraphKit::type_check(Node* recv_klass, const TypeKlassPtr* tklass,
3082                            float prob) {
3083   Node* want_klass = makecon(tklass);
3084   Node* cmp = _gvn.transform(new CmpPNode(recv_klass, want_klass));
3085   Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
3086   IfNode* iff = create_and_xform_if(control(), bol, prob, COUNT_UNKNOWN);
3087   set_control(_gvn.transform(new IfTrueNode (iff)));
3088   Node* fail = _gvn.transform(new IfFalseNode(iff));
3089   return fail;
3090 }
3091 
3092 //------------------------------subtype_check_receiver-------------------------
3093 Node* GraphKit::subtype_check_receiver(Node* receiver, ciKlass* klass,
3094                                        Node** casted_receiver) {
3095   const TypeKlassPtr* tklass = TypeKlassPtr::make(klass, Type::trust_interfaces)->try_improve();
3096   Node* want_klass = makecon(tklass);
3097 
3098   Node* slow_ctl = gen_subtype_check(receiver, want_klass);
3099 
3100   // Ignore interface type information until interface types are properly tracked.
3101   if (!stopped() && !klass->is_interface()) {
3102     const TypeOopPtr* receiver_type = _gvn.type(receiver)->isa_oopptr();
3103     const TypeOopPtr* recv_type = tklass->cast_to_exactness(false)->is_klassptr()->as_instance_type();
3104     if (receiver_type != nullptr && !receiver_type->higher_equal(recv_type)) { // ignore redundant casts
3105       Node* cast = _gvn.transform(new CheckCastPPNode(control(), receiver, recv_type));
3106       if (recv_type->is_inlinetypeptr()) {
3107         cast = InlineTypeNode::make_from_oop(this, cast, recv_type->inline_klass());
3108       }
3109       (*casted_receiver) = cast;
3110     }
3111   }
3112 
3113   return slow_ctl;
3114 }
3115 
3116 //------------------------------seems_never_null-------------------------------
3117 // Use null_seen information if it is available from the profile.
3118 // If we see an unexpected null at a type check we record it and force a
3119 // recompile; the offending check will be recompiled to handle nulls.
3120 // If we see several offending BCIs, then all checks in the
3121 // method will be recompiled.
3122 bool GraphKit::seems_never_null(Node* obj, ciProfileData* data, bool& speculating) {
3123   speculating = !_gvn.type(obj)->speculative_maybe_null();
3124   Deoptimization::DeoptReason reason = Deoptimization::reason_null_check(speculating);
3125   if (UncommonNullCast               // Cutout for this technique
3126       && obj != null()               // And not the -Xcomp stupid case?
3127       && !too_many_traps(reason)
3128       ) {
3129     if (speculating) {

3198 
3199 //------------------------maybe_cast_profiled_receiver-------------------------
3200 // If the profile has seen exactly one type, narrow to exactly that type.
3201 // Subsequent type checks will always fold up.
3202 Node* GraphKit::maybe_cast_profiled_receiver(Node* not_null_obj,
3203                                              const TypeKlassPtr* require_klass,
3204                                              ciKlass* spec_klass,
3205                                              bool safe_for_replace) {
3206   if (!UseTypeProfile || !TypeProfileCasts) return nullptr;
3207 
3208   Deoptimization::DeoptReason reason = Deoptimization::reason_class_check(spec_klass != nullptr);
3209 
3210   // Make sure we haven't already deoptimized from this tactic.
3211   if (too_many_traps_or_recompiles(reason))
3212     return nullptr;
3213 
3214   // (No, this isn't a call, but it's enough like a virtual call
3215   // to use the same ciMethod accessor to get the profile info...)
3216   // If we have a speculative type use it instead of profiling (which
3217   // may not help us)
3218   ciKlass* exact_kls = spec_klass;
3219   if (exact_kls == nullptr) {
3220     if (java_bc() == Bytecodes::_aastore) {
3221       ciKlass* array_type = nullptr;
3222       ciKlass* element_type = nullptr;
3223       ProfilePtrKind element_ptr = ProfileMaybeNull;
3224       bool flat_array = true;
3225       bool null_free_array = true;
3226       method()->array_access_profiled_type(bci(), array_type, element_type, element_ptr, flat_array, null_free_array);
3227       exact_kls = element_type;
3228     } else {
3229       exact_kls = profile_has_unique_klass();
3230     }
3231   }
3232   if (exact_kls != nullptr) {// no cast failures here
3233     if (require_klass == nullptr ||
3234         C->static_subtype_check(require_klass, TypeKlassPtr::make(exact_kls, Type::trust_interfaces)) == Compile::SSC_always_true) {
3235       // If we narrow the type to match what the type profile sees or
3236       // the speculative type, we can then remove the rest of the
3237       // cast.
3238       // This is a win, even if the exact_kls is very specific,
3239       // because downstream operations, such as method calls,
3240       // will often benefit from the sharper type.
3241       Node* exact_obj = not_null_obj; // will get updated in place...
3242       Node* slow_ctl  = type_check_receiver(exact_obj, exact_kls, 1.0,
3243                                             &exact_obj);
3244       { PreserveJVMState pjvms(this);
3245         set_control(slow_ctl);
3246         uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
3247       }
3248       if (safe_for_replace) {
3249         replace_in_map(not_null_obj, exact_obj);
3250       }
3251       return exact_obj;

3341   // If not_null_obj is dead, only null-path is taken
3342   if (stopped()) {              // Doing instance-of on a null?
3343     set_control(null_ctl);
3344     return intcon(0);
3345   }
3346   region->init_req(_null_path, null_ctl);
3347   phi   ->init_req(_null_path, intcon(0)); // Set null path value
3348   if (null_ctl == top()) {
3349     // Do this eagerly, so that pattern matches like is_diamond_phi
3350     // will work even during parsing.
3351     assert(_null_path == PATH_LIMIT-1, "delete last");
3352     region->del_req(_null_path);
3353     phi   ->del_req(_null_path);
3354   }
3355 
3356   // Do we know the type check always succeed?
3357   bool known_statically = false;
3358   if (_gvn.type(superklass)->singleton()) {
3359     const TypeKlassPtr* superk = _gvn.type(superklass)->is_klassptr();
3360     const TypeKlassPtr* subk = _gvn.type(obj)->is_oopptr()->as_klass_type();
3361     if (subk != nullptr && subk->is_loaded()) {
3362       int static_res = C->static_subtype_check(superk, subk);
3363       known_statically = (static_res == Compile::SSC_always_true || static_res == Compile::SSC_always_false);
3364     }
3365   }
3366 
3367   if (!known_statically) {
3368     const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr();
3369     // We may not have profiling here or it may not help us. If we
3370     // have a speculative type use it to perform an exact cast.
3371     ciKlass* spec_obj_type = obj_type->speculative_type();
3372     if (spec_obj_type != nullptr || (ProfileDynamicTypes && data != nullptr)) {
3373       Node* cast_obj = maybe_cast_profiled_receiver(not_null_obj, nullptr, spec_obj_type, safe_for_replace);
3374       if (stopped()) {            // Profile disagrees with this path.
3375         set_control(null_ctl);    // Null is the only remaining possibility.
3376         return intcon(0);
3377       }
3378       if (cast_obj != nullptr) {
3379         not_null_obj = cast_obj;
3380       }
3381     }

3397   record_for_igvn(region);
3398 
3399   // If we know the type check always succeeds then we don't use the
3400   // profiling data at this bytecode. Don't lose it, feed it to the
3401   // type system as a speculative type.
3402   if (safe_for_replace) {
3403     Node* casted_obj = record_profiled_receiver_for_speculation(obj);
3404     replace_in_map(obj, casted_obj);
3405   }
3406 
3407   return _gvn.transform(phi);
3408 }
3409 
3410 //-------------------------------gen_checkcast---------------------------------
3411 // Generate a checkcast idiom.  Used by both the checkcast bytecode and the
3412 // array store bytecode.  Stack must be as-if BEFORE doing the bytecode so the
3413 // uncommon-trap paths work.  Adjust stack after this call.
3414 // If failure_control is supplied and not null, it is filled in with
3415 // the control edge for the cast failure.  Otherwise, an appropriate
3416 // uncommon trap or exception is thrown.
3417 Node* GraphKit::gen_checkcast(Node *obj, Node* superklass, Node* *failure_control, bool null_free) {

3418   kill_dead_locals();           // Benefit all the uncommon traps
3419   const TypeKlassPtr *tk = _gvn.type(superklass)->is_klassptr()->try_improve();
3420   const TypeOopPtr *toop = tk->cast_to_exactness(false)->as_instance_type();
3421   bool safe_for_replace = (failure_control == nullptr);
3422   assert(!null_free || toop->is_inlinetypeptr(), "must be an inline type pointer");
3423 
3424   // Fast cutout:  Check the case that the cast is vacuously true.
3425   // This detects the common cases where the test will short-circuit
3426   // away completely.  We do this before we perform the null check,
3427   // because if the test is going to turn into zero code, we don't
3428   // want a residual null check left around.  (Causes a slowdown,
3429   // for example, in some objArray manipulations, such as a[i]=a[j].)
3430   if (tk->singleton()) {
3431     const TypeKlassPtr* kptr = nullptr;
3432     const Type* t = _gvn.type(obj);
3433     if (t->isa_oop_ptr()) {
3434       kptr = t->is_oopptr()->as_klass_type();
3435     } else if (obj->is_InlineType()) {
3436       ciInlineKlass* vk = t->inline_klass();
3437       kptr = TypeInstKlassPtr::make(TypePtr::NotNull, vk, Type::Offset(0));
3438     }
3439     if (kptr != nullptr) {
3440       switch (C->static_subtype_check(tk, kptr)) {
3441       case Compile::SSC_always_true:
3442         // If we know the type check always succeed then we don't use
3443         // the profiling data at this bytecode. Don't lose it, feed it
3444         // to the type system as a speculative type.
3445         obj = record_profiled_receiver_for_speculation(obj);
3446         if (null_free) {
3447           assert(safe_for_replace, "must be");
3448           obj = null_check(obj);
3449         }
3450         assert(stopped() || !toop->is_inlinetypeptr() || obj->is_InlineType(), "should have been scalarized");
3451         return obj;
3452       case Compile::SSC_always_false:
3453         if (null_free) {
3454           assert(safe_for_replace, "must be");
3455           obj = null_check(obj);
3456         }
3457         // It needs a null check because a null will *pass* the cast check.
3458         if (t->isa_oopptr() != nullptr && !t->is_oopptr()->maybe_null()) {

3459           bool is_aastore = (java_bc() == Bytecodes::_aastore);
3460           Deoptimization::DeoptReason reason = is_aastore ?
3461             Deoptimization::Reason_array_check : Deoptimization::Reason_class_check;
3462           builtin_throw(reason);
3463           return top();
3464         } else if (!too_many_traps_or_recompiles(Deoptimization::Reason_null_assert)) {
3465           return null_assert(obj);
3466         }
3467         break; // Fall through to full check
3468       default:
3469         break;
3470       }
3471     }
3472   }
3473 
3474   ciProfileData* data = nullptr;

3475   if (failure_control == nullptr) {        // use MDO in regular case only
3476     assert(java_bc() == Bytecodes::_aastore ||
3477            java_bc() == Bytecodes::_checkcast,
3478            "interpreter profiles type checks only for these BCs");
3479     if (method()->method_data()->is_mature()) {
3480       data = method()->method_data()->bci_to_data(bci());
3481     }
3482   }
3483 
3484   // Make the merge point
3485   enum { _obj_path = 1, _null_path, PATH_LIMIT };
3486   RegionNode* region = new RegionNode(PATH_LIMIT);
3487   Node*       phi    = new PhiNode(region, toop);
3488   _gvn.set_type(region, Type::CONTROL);
3489   _gvn.set_type(phi, toop);
3490 
3491   C->set_has_split_ifs(true); // Has chance for split-if optimization
3492 
3493   // Use null-cast information if it is available
3494   bool speculative_not_null = false;
3495   bool never_see_null = ((failure_control == nullptr)  // regular case only
3496                          && seems_never_null(obj, data, speculative_not_null));
3497 
3498   if (obj->is_InlineType()) {
3499     // Re-execute if buffering during triggers deoptimization
3500     PreserveReexecuteState preexecs(this);
3501     jvms()->set_should_reexecute(true);
3502     obj = obj->as_InlineType()->buffer(this, safe_for_replace);
3503   }
3504 
3505   // Null check; get casted pointer; set region slot 3
3506   Node* null_ctl = top();
3507   Node* not_null_obj = nullptr;
3508   if (null_free) {
3509     assert(safe_for_replace, "must be");
3510     not_null_obj = null_check(obj);
3511   } else {
3512     not_null_obj = null_check_oop(obj, &null_ctl, never_see_null, safe_for_replace, speculative_not_null);
3513   }
3514 
3515   // If not_null_obj is dead, only null-path is taken
3516   if (stopped()) {              // Doing instance-of on a null?
3517     set_control(null_ctl);
3518     if (toop->is_inlinetypeptr()) {
3519       return InlineTypeNode::make_null(_gvn, toop->inline_klass());
3520     }
3521     return null();
3522   }
3523   region->init_req(_null_path, null_ctl);
3524   phi   ->init_req(_null_path, null());  // Set null path value
3525   if (null_ctl == top()) {
3526     // Do this eagerly, so that pattern matches like is_diamond_phi
3527     // will work even during parsing.
3528     assert(_null_path == PATH_LIMIT-1, "delete last");
3529     region->del_req(_null_path);
3530     phi   ->del_req(_null_path);
3531   }
3532 
3533   Node* cast_obj = nullptr;
3534   if (tk->klass_is_exact()) {
3535     // The following optimization tries to statically cast the speculative type of the object
3536     // (for example obtained during profiling) to the type of the superklass and then do a
3537     // dynamic check that the type of the object is what we expect. To work correctly
3538     // for checkcast and aastore the type of superklass should be exact.
3539     const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr();
3540     // We may not have profiling here or it may not help us. If we have
3541     // a speculative type use it to perform an exact cast.
3542     ciKlass* spec_obj_type = obj_type->speculative_type();
3543     if (spec_obj_type != nullptr || data != nullptr) {
3544       cast_obj = maybe_cast_profiled_receiver(not_null_obj, tk, spec_obj_type, safe_for_replace);
3545       if (cast_obj != nullptr) {
3546         if (failure_control != nullptr) // failure is now impossible
3547           (*failure_control) = top();
3548         // adjust the type of the phi to the exact klass:
3549         phi->raise_bottom_type(_gvn.type(cast_obj)->meet_speculative(TypePtr::NULL_PTR));
3550       }
3551     }
3552   }
3553 
3554   if (cast_obj == nullptr) {
3555     // Generate the subtype check
3556     Node* not_subtype_ctrl = gen_subtype_check(not_null_obj, superklass);
3557 
3558     // Plug in success path into the merge
3559     cast_obj = _gvn.transform(new CheckCastPPNode(control(), not_null_obj, toop));
3560     // Failure path ends in uncommon trap (or may be dead - failure impossible)
3561     if (failure_control == nullptr) {
3562       if (not_subtype_ctrl != top()) { // If failure is possible
3563         PreserveJVMState pjvms(this);
3564         set_control(not_subtype_ctrl);
3565         Node* obj_klass = nullptr;
3566         if (not_null_obj->is_InlineType()) {
3567           obj_klass = makecon(TypeKlassPtr::make(_gvn.type(not_null_obj)->inline_klass()));
3568         } else {
3569           obj_klass = load_object_klass(not_null_obj);
3570         }
3571         bool is_aastore = (java_bc() == Bytecodes::_aastore);
3572         Deoptimization::DeoptReason reason = is_aastore ?
3573           Deoptimization::Reason_array_check : Deoptimization::Reason_class_check;
3574         builtin_throw(reason);
3575       }
3576     } else {
3577       (*failure_control) = not_subtype_ctrl;
3578     }
3579   }
3580 
3581   region->init_req(_obj_path, control());
3582   phi   ->init_req(_obj_path, cast_obj);
3583 
3584   // A merge of null or Casted-NotNull obj
3585   Node* res = _gvn.transform(phi);
3586 
3587   // Note I do NOT always 'replace_in_map(obj,result)' here.
3588   //  if( tk->klass()->can_be_primary_super()  )
3589     // This means that if I successfully store an Object into an array-of-String
3590     // I 'forget' that the Object is really now known to be a String.  I have to
3591     // do this because we don't have true union types for interfaces - if I store
3592     // a Baz into an array-of-Interface and then tell the optimizer it's an
3593     // Interface, I forget that it's also a Baz and cannot do Baz-like field
3594     // references to it.  FIX THIS WHEN UNION TYPES APPEAR!
3595   //  replace_in_map( obj, res );
3596 
3597   // Return final merged results
3598   set_control( _gvn.transform(region) );
3599   record_for_igvn(region);
3600 
3601   bool not_inline = !toop->can_be_inline_type();
3602   bool not_flat_in_array = !UseFlatArray || not_inline || (toop->is_inlinetypeptr() && !toop->inline_klass()->flat_in_array());
3603   if (EnableValhalla && not_flat_in_array) {
3604     // Check if obj has been loaded from an array
3605     obj = obj->isa_DecodeN() ? obj->in(1) : obj;
3606     Node* array = nullptr;
3607     if (obj->isa_Load()) {
3608       Node* address = obj->in(MemNode::Address);
3609       if (address->isa_AddP()) {
3610         array = address->as_AddP()->in(AddPNode::Base);
3611       }
3612     } else if (obj->is_Phi()) {
3613       Node* region = obj->in(0);
3614       // TODO make this more robust (see JDK-8231346)
3615       if (region->req() == 3 && region->in(2) != nullptr && region->in(2)->in(0) != nullptr) {
3616         IfNode* iff = region->in(2)->in(0)->isa_If();
3617         if (iff != nullptr) {
3618           iff->is_flat_array_check(&_gvn, &array);
3619         }
3620       }
3621     }
3622     if (array != nullptr) {
3623       const TypeAryPtr* ary_t = _gvn.type(array)->isa_aryptr();
3624       if (ary_t != nullptr && !ary_t->is_flat()) {
3625         if (!ary_t->is_not_null_free() && not_inline) {
3626           // Casting array element to a non-inline-type, mark array as not null-free.
3627           Node* cast = _gvn.transform(new CheckCastPPNode(control(), array, ary_t->cast_to_not_null_free()));
3628           replace_in_map(array, cast);
3629         } else if (!ary_t->is_not_flat()) {
3630           // Casting array element to a non-flat type, mark array as not flat.
3631           Node* cast = _gvn.transform(new CheckCastPPNode(control(), array, ary_t->cast_to_not_flat()));
3632           replace_in_map(array, cast);
3633         }
3634       }
3635     }
3636   }
3637 
3638   if (!stopped() && !res->is_InlineType()) {
3639     res = record_profiled_receiver_for_speculation(res);
3640     if (toop->is_inlinetypeptr()) {
3641       Node* vt = InlineTypeNode::make_from_oop(this, res, toop->inline_klass(), !gvn().type(res)->maybe_null());
3642       res = vt;
3643       if (safe_for_replace) {
3644         replace_in_map(obj, vt);
3645         replace_in_map(not_null_obj, vt);
3646         replace_in_map(res, vt);
3647       }
3648     }
3649   }
3650   return res;
3651 }
3652 
3653 Node* GraphKit::inline_type_test(Node* obj, bool is_inline) {
3654   Node* mark_adr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
3655   Node* mark = make_load(nullptr, mark_adr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered);
3656   Node* mask = MakeConX(markWord::inline_type_pattern);
3657   Node* masked = _gvn.transform(new AndXNode(mark, mask));
3658   Node* cmp = _gvn.transform(new CmpXNode(masked, mask));
3659   return _gvn.transform(new BoolNode(cmp, is_inline ? BoolTest::eq : BoolTest::ne));
3660 }
3661 
3662 Node* GraphKit::is_val_mirror(Node* mirror) {
3663   // JDK-8325660: notion of secondary mirror / val_mirror is gone one JEP 401
3664   Node* p = basic_plus_adr(mirror, (int)0 /* java_lang_Class::secondary_mirror_offset() */);
3665   Node* secondary_mirror = access_load_at(mirror, p, _gvn.type(p)->is_ptr(), TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR), T_OBJECT, IN_HEAP);
3666   Node* cmp = _gvn.transform(new CmpPNode(mirror, secondary_mirror));
3667   return _gvn.transform(new BoolNode(cmp, BoolTest::eq));
3668 }
3669 
3670 Node* GraphKit::array_lh_test(Node* klass, jint mask, jint val, bool eq) {
3671   Node* lh_adr = basic_plus_adr(klass, in_bytes(Klass::layout_helper_offset()));
3672   // Make sure to use immutable memory here to enable hoisting the check out of loops
3673   Node* lh_val = _gvn.transform(LoadNode::make(_gvn, nullptr, immutable_memory(), lh_adr, lh_adr->bottom_type()->is_ptr(), TypeInt::INT, T_INT, MemNode::unordered));
3674   Node* masked = _gvn.transform(new AndINode(lh_val, intcon(mask)));
3675   Node* cmp = _gvn.transform(new CmpINode(masked, intcon(val)));
3676   return _gvn.transform(new BoolNode(cmp, eq ? BoolTest::eq : BoolTest::ne));
3677 }
3678 
3679 Node* GraphKit::flat_array_test(Node* array_or_klass, bool flat) {
3680   // We can't use immutable memory here because the mark word is mutable.
3681   // PhaseIdealLoop::move_flat_array_check_out_of_loop will make sure the
3682   // check is moved out of loops (mainly to enable loop unswitching).
3683   Node* mem = UseArrayMarkWordCheck ? memory(Compile::AliasIdxRaw) : immutable_memory();
3684   Node* cmp = _gvn.transform(new FlatArrayCheckNode(C, mem, array_or_klass));
3685   record_for_igvn(cmp); // Give it a chance to be optimized out by IGVN
3686   return _gvn.transform(new BoolNode(cmp, flat ? BoolTest::eq : BoolTest::ne));
3687 }
3688 
3689 Node* GraphKit::null_free_array_test(Node* klass, bool null_free) {
3690   return array_lh_test(klass, Klass::_lh_null_free_array_bit_inplace, 0, !null_free);
3691 }
3692 
3693 // Deoptimize if 'ary' is a null-free inline type array and 'val' is null
3694 Node* GraphKit::inline_array_null_guard(Node* ary, Node* val, int nargs, bool safe_for_replace) {
3695   RegionNode* region = new RegionNode(3);
3696   Node* null_ctl = top();
3697   null_check_oop(val, &null_ctl);
3698   if (null_ctl != top()) {
3699     PreserveJVMState pjvms(this);
3700     set_control(null_ctl);
3701     {
3702       // Deoptimize if null-free array
3703       BuildCutout unless(this, null_free_array_test(load_object_klass(ary), /* null_free = */ false), PROB_MAX);
3704       inc_sp(nargs);
3705       uncommon_trap(Deoptimization::Reason_null_check,
3706                     Deoptimization::Action_none);
3707     }
3708     region->init_req(1, control());
3709   }
3710   region->init_req(2, control());
3711   set_control(_gvn.transform(region));
3712   record_for_igvn(region);
3713   if (_gvn.type(val) == TypePtr::NULL_PTR) {
3714     // Since we were just successfully storing null, the array can't be null free.
3715     const TypeAryPtr* ary_t = _gvn.type(ary)->is_aryptr();
3716     ary_t = ary_t->cast_to_not_null_free();
3717     Node* cast = _gvn.transform(new CheckCastPPNode(control(), ary, ary_t));
3718     if (safe_for_replace) {
3719       replace_in_map(ary, cast);
3720     }
3721     ary = cast;
3722   }
3723   return ary;
3724 }
3725 
3726 //------------------------------next_monitor-----------------------------------
3727 // What number should be given to the next monitor?
3728 int GraphKit::next_monitor() {
3729   int current = jvms()->monitor_depth()* C->sync_stack_slots();
3730   int next = current + C->sync_stack_slots();
3731   // Keep the toplevel high water mark current:
3732   if (C->fixed_slots() < next)  C->set_fixed_slots(next);
3733   return current;
3734 }
3735 
3736 //------------------------------insert_mem_bar---------------------------------
3737 // Memory barrier to avoid floating things around
3738 // The membar serves as a pinch point between both control and all memory slices.
3739 Node* GraphKit::insert_mem_bar(int opcode, Node* precedent) {
3740   MemBarNode* mb = MemBarNode::make(C, opcode, Compile::AliasIdxBot, precedent);
3741   mb->init_req(TypeFunc::Control, control());
3742   mb->init_req(TypeFunc::Memory,  reset_memory());
3743   Node* membar = _gvn.transform(mb);

3771   }
3772   Node* membar = _gvn.transform(mb);
3773   set_control(_gvn.transform(new ProjNode(membar, TypeFunc::Control)));
3774   if (alias_idx == Compile::AliasIdxBot) {
3775     merged_memory()->set_base_memory(_gvn.transform(new ProjNode(membar, TypeFunc::Memory)));
3776   } else {
3777     set_memory(_gvn.transform(new ProjNode(membar, TypeFunc::Memory)),alias_idx);
3778   }
3779   return membar;
3780 }
3781 
3782 //------------------------------shared_lock------------------------------------
3783 // Emit locking code.
3784 FastLockNode* GraphKit::shared_lock(Node* obj) {
3785   // bci is either a monitorenter bc or InvocationEntryBci
3786   // %%% SynchronizationEntryBCI is redundant; use InvocationEntryBci in interfaces
3787   assert(SynchronizationEntryBCI == InvocationEntryBci, "");
3788 
3789   if( !GenerateSynchronizationCode )
3790     return nullptr;                // Not locking things?
3791 
3792   if (stopped())                // Dead monitor?
3793     return nullptr;
3794 
3795   assert(dead_locals_are_killed(), "should kill locals before sync. point");
3796 
3797   // Box the stack location
3798   Node* box = _gvn.transform(new BoxLockNode(next_monitor()));
3799   Node* mem = reset_memory();
3800 
3801   FastLockNode * flock = _gvn.transform(new FastLockNode(0, obj, box) )->as_FastLock();
3802 
3803   // Create the rtm counters for this fast lock if needed.
3804   flock->create_rtm_lock_counter(sync_jvms()); // sync_jvms used to get current bci
3805 
3806   // Add monitor to debug info for the slow path.  If we block inside the
3807   // slow path and de-opt, we need the monitor hanging around
3808   map()->push_monitor( flock );
3809 
3810   const TypeFunc *tf = LockNode::lock_type();
3811   LockNode *lock = new LockNode(C, tf);

3840   }
3841 #endif
3842 
3843   return flock;
3844 }
3845 
3846 
3847 //------------------------------shared_unlock----------------------------------
3848 // Emit unlocking code.
3849 void GraphKit::shared_unlock(Node* box, Node* obj) {
3850   // bci is either a monitorenter bc or InvocationEntryBci
3851   // %%% SynchronizationEntryBCI is redundant; use InvocationEntryBci in interfaces
3852   assert(SynchronizationEntryBCI == InvocationEntryBci, "");
3853 
3854   if( !GenerateSynchronizationCode )
3855     return;
3856   if (stopped()) {               // Dead monitor?
3857     map()->pop_monitor();        // Kill monitor from debug info
3858     return;
3859   }
3860   assert(!obj->is_InlineType(), "should not unlock on inline type");
3861 
3862   // Memory barrier to avoid floating things down past the locked region
3863   insert_mem_bar(Op_MemBarReleaseLock);
3864 
3865   const TypeFunc *tf = OptoRuntime::complete_monitor_exit_Type();
3866   UnlockNode *unlock = new UnlockNode(C, tf);
3867 #ifdef ASSERT
3868   unlock->set_dbg_jvms(sync_jvms());
3869 #endif
3870   uint raw_idx = Compile::AliasIdxRaw;
3871   unlock->init_req( TypeFunc::Control, control() );
3872   unlock->init_req( TypeFunc::Memory , memory(raw_idx) );
3873   unlock->init_req( TypeFunc::I_O    , top() )     ;   // does no i/o
3874   unlock->init_req( TypeFunc::FramePtr, frameptr() );
3875   unlock->init_req( TypeFunc::ReturnAdr, top() );
3876 
3877   unlock->init_req(TypeFunc::Parms + 0, obj);
3878   unlock->init_req(TypeFunc::Parms + 1, box);
3879   unlock = _gvn.transform(unlock)->as_Unlock();
3880 
3881   Node* mem = reset_memory();
3882 
3883   // unlock has no side-effects, sets few values
3884   set_predefined_output_for_runtime_call(unlock, mem, TypeRawPtr::BOTTOM);
3885 
3886   // Kill monitor from debug info
3887   map()->pop_monitor( );
3888 }
3889 
3890 //-------------------------------get_layout_helper-----------------------------
3891 // If the given klass is a constant or known to be an array,
3892 // fetch the constant layout helper value into constant_value
3893 // and return null.  Otherwise, load the non-constant
3894 // layout helper value, and return the node which represents it.
3895 // This two-faced routine is useful because allocation sites
3896 // almost always feature constant types.
3897 Node* GraphKit::get_layout_helper(Node* klass_node, jint& constant_value) {
3898   const TypeKlassPtr* klass_t = _gvn.type(klass_node)->isa_klassptr();
3899   if (!StressReflectiveCode && klass_t != nullptr) {
3900     bool xklass = klass_t->klass_is_exact();
3901     bool can_be_flat = false;
3902     const TypeAryPtr* ary_type = klass_t->as_instance_type()->isa_aryptr();
3903     if (UseFlatArray && !xklass && ary_type != nullptr && !ary_type->is_null_free()) {
3904       // The runtime type of [LMyValue might be [QMyValue due to [QMyValue <: [LMyValue. Don't constant fold.
3905       const TypeOopPtr* elem = ary_type->elem()->make_oopptr();
3906       can_be_flat = ary_type->can_be_inline_array() && (!elem->is_inlinetypeptr() || elem->inline_klass()->flat_in_array());
3907     }
3908     if (!can_be_flat && (xklass || (klass_t->isa_aryklassptr() && klass_t->is_aryklassptr()->elem() != Type::BOTTOM))) {
3909       jint lhelper;
3910       if (klass_t->is_flat()) {
3911         lhelper = ary_type->flat_layout_helper();
3912       } else if (klass_t->isa_aryklassptr()) {
3913         BasicType elem = ary_type->elem()->array_element_basic_type();
3914         if (is_reference_type(elem, true)) {
3915           elem = T_OBJECT;
3916         }
3917         lhelper = Klass::array_layout_helper(elem);
3918       } else {
3919         lhelper = klass_t->is_instklassptr()->exact_klass()->layout_helper();
3920       }
3921       if (lhelper != Klass::_lh_neutral_value) {
3922         constant_value = lhelper;
3923         return (Node*) nullptr;
3924       }
3925     }
3926   }
3927   constant_value = Klass::_lh_neutral_value;  // put in a known value
3928   Node* lhp = basic_plus_adr(klass_node, klass_node, in_bytes(Klass::layout_helper_offset()));
3929   return make_load(nullptr, lhp, TypeInt::INT, T_INT, MemNode::unordered);
3930 }
3931 
3932 // We just put in an allocate/initialize with a big raw-memory effect.
3933 // Hook selected additional alias categories on the initialization.
3934 static void hook_memory_on_init(GraphKit& kit, int alias_idx,
3935                                 MergeMemNode* init_in_merge,
3936                                 Node* init_out_raw) {
3937   DEBUG_ONLY(Node* init_in_raw = init_in_merge->base_memory());
3938   assert(init_in_merge->memory_at(alias_idx) == init_in_raw, "");
3939 
3940   Node* prevmem = kit.memory(alias_idx);
3941   init_in_merge->set_memory_at(alias_idx, prevmem);
3942   if (init_out_raw != nullptr) {
3943     kit.set_memory(init_out_raw, alias_idx);
3944   }
3945 }
3946 
3947 //---------------------------set_output_for_allocation-------------------------
3948 Node* GraphKit::set_output_for_allocation(AllocateNode* alloc,
3949                                           const TypeOopPtr* oop_type,
3950                                           bool deoptimize_on_exception) {
3951   int rawidx = Compile::AliasIdxRaw;
3952   alloc->set_req( TypeFunc::FramePtr, frameptr() );
3953   add_safepoint_edges(alloc);
3954   Node* allocx = _gvn.transform(alloc);
3955   set_control( _gvn.transform(new ProjNode(allocx, TypeFunc::Control) ) );
3956   // create memory projection for i_o
3957   set_memory ( _gvn.transform( new ProjNode(allocx, TypeFunc::Memory, true) ), rawidx );
3958   make_slow_call_ex(allocx, env()->Throwable_klass(), true, deoptimize_on_exception);
3959 
3960   // create a memory projection as for the normal control path
3961   Node* malloc = _gvn.transform(new ProjNode(allocx, TypeFunc::Memory));
3962   set_memory(malloc, rawidx);
3963 
3964   // a normal slow-call doesn't change i_o, but an allocation does
3965   // we create a separate i_o projection for the normal control path
3966   set_i_o(_gvn.transform( new ProjNode(allocx, TypeFunc::I_O, false) ) );
3967   Node* rawoop = _gvn.transform( new ProjNode(allocx, TypeFunc::Parms) );
3968 
3969   // put in an initialization barrier
3970   InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, rawidx,
3971                                                  rawoop)->as_Initialize();
3972   assert(alloc->initialization() == init,  "2-way macro link must work");
3973   assert(init ->allocation()     == alloc, "2-way macro link must work");
3974   {
3975     // Extract memory strands which may participate in the new object's
3976     // initialization, and source them from the new InitializeNode.
3977     // This will allow us to observe initializations when they occur,
3978     // and link them properly (as a group) to the InitializeNode.
3979     assert(init->in(InitializeNode::Memory) == malloc, "");
3980     MergeMemNode* minit_in = MergeMemNode::make(malloc);
3981     init->set_req(InitializeNode::Memory, minit_in);
3982     record_for_igvn(minit_in); // fold it up later, if possible
3983     _gvn.set_type(minit_in, Type::MEMORY);
3984     Node* minit_out = memory(rawidx);
3985     assert(minit_out->is_Proj() && minit_out->in(0) == init, "");
3986     // Add an edge in the MergeMem for the header fields so an access
3987     // to one of those has correct memory state
3988     set_memory(minit_out, C->get_alias_index(oop_type->add_offset(oopDesc::mark_offset_in_bytes())));
3989     set_memory(minit_out, C->get_alias_index(oop_type->add_offset(oopDesc::klass_offset_in_bytes())));
3990     if (oop_type->isa_aryptr()) {
3991       const TypeAryPtr* arytype = oop_type->is_aryptr();
3992       if (arytype->is_flat()) {
3993         // Initially all flat array accesses share a single slice
3994         // but that changes after parsing. Prepare the memory graph so
3995         // it can optimize flat array accesses properly once they
3996         // don't share a single slice.
3997         assert(C->flat_accesses_share_alias(), "should be set at parse time");
3998         C->set_flat_accesses_share_alias(false);
3999         ciInlineKlass* vk = arytype->elem()->inline_klass();
4000         for (int i = 0, len = vk->nof_nonstatic_fields(); i < len; i++) {
4001           ciField* field = vk->nonstatic_field_at(i);
4002           if (field->offset_in_bytes() >= TrackedInitializationLimit * HeapWordSize)
4003             continue;  // do not bother to track really large numbers of fields
4004           int off_in_vt = field->offset_in_bytes() - vk->first_field_offset();
4005           const TypePtr* adr_type = arytype->with_field_offset(off_in_vt)->add_offset(Type::OffsetBot);
4006           int fieldidx = C->get_alias_index(adr_type, true);
4007           // Pass nullptr for init_out. Having per flat array element field memory edges as uses of the Initialize node
4008           // can result in per flat array field Phis to be created which confuses the logic of
4009           // Compile::adjust_flat_array_access_aliases().
4010           hook_memory_on_init(*this, fieldidx, minit_in, nullptr);
4011         }
4012         C->set_flat_accesses_share_alias(true);
4013         hook_memory_on_init(*this, C->get_alias_index(TypeAryPtr::INLINES), minit_in, minit_out);
4014       } else {
4015         const TypePtr* telemref = oop_type->add_offset(Type::OffsetBot);
4016         int            elemidx  = C->get_alias_index(telemref);
4017         hook_memory_on_init(*this, elemidx, minit_in, minit_out);
4018       }
4019     } else if (oop_type->isa_instptr()) {
4020       set_memory(minit_out, C->get_alias_index(oop_type)); // mark word
4021       ciInstanceKlass* ik = oop_type->is_instptr()->instance_klass();
4022       for (int i = 0, len = ik->nof_nonstatic_fields(); i < len; i++) {
4023         ciField* field = ik->nonstatic_field_at(i);
4024         if (field->offset_in_bytes() >= TrackedInitializationLimit * HeapWordSize)
4025           continue;  // do not bother to track really large numbers of fields
4026         // Find (or create) the alias category for this field:
4027         int fieldidx = C->alias_type(field)->index();
4028         hook_memory_on_init(*this, fieldidx, minit_in, minit_out);
4029       }
4030     }
4031   }
4032 
4033   // Cast raw oop to the real thing...
4034   Node* javaoop = new CheckCastPPNode(control(), rawoop, oop_type);
4035   javaoop = _gvn.transform(javaoop);
4036   C->set_recent_alloc(control(), javaoop);
4037   assert(just_allocated_object(control()) == javaoop, "just allocated");
4038 
4039 #ifdef ASSERT
4040   { // Verify that the AllocateNode::Ideal_allocation recognizers work:

4051       assert(alloc->in(AllocateNode::ALength)->is_top(), "no length, please");
4052     }
4053   }
4054 #endif //ASSERT
4055 
4056   return javaoop;
4057 }
4058 
4059 //---------------------------new_instance--------------------------------------
4060 // This routine takes a klass_node which may be constant (for a static type)
4061 // or may be non-constant (for reflective code).  It will work equally well
4062 // for either, and the graph will fold nicely if the optimizer later reduces
4063 // the type to a constant.
4064 // The optional arguments are for specialized use by intrinsics:
4065 //  - If 'extra_slow_test' if not null is an extra condition for the slow-path.
4066 //  - If 'return_size_val', report the total object size to the caller.
4067 //  - deoptimize_on_exception controls how Java exceptions are handled (rethrow vs deoptimize)
4068 Node* GraphKit::new_instance(Node* klass_node,
4069                              Node* extra_slow_test,
4070                              Node* *return_size_val,
4071                              bool deoptimize_on_exception,
4072                              InlineTypeNode* inline_type_node) {
4073   // Compute size in doublewords
4074   // The size is always an integral number of doublewords, represented
4075   // as a positive bytewise size stored in the klass's layout_helper.
4076   // The layout_helper also encodes (in a low bit) the need for a slow path.
4077   jint  layout_con = Klass::_lh_neutral_value;
4078   Node* layout_val = get_layout_helper(klass_node, layout_con);
4079   bool  layout_is_con = (layout_val == nullptr);
4080 
4081   if (extra_slow_test == nullptr)  extra_slow_test = intcon(0);
4082   // Generate the initial go-slow test.  It's either ALWAYS (return a
4083   // Node for 1) or NEVER (return a null) or perhaps (in the reflective
4084   // case) a computed value derived from the layout_helper.
4085   Node* initial_slow_test = nullptr;
4086   if (layout_is_con) {
4087     assert(!StressReflectiveCode, "stress mode does not use these paths");
4088     bool must_go_slow = Klass::layout_helper_needs_slow_path(layout_con);
4089     initial_slow_test = must_go_slow ? intcon(1) : extra_slow_test;
4090   } else {   // reflective case
4091     // This reflective path is used by Unsafe.allocateInstance.
4092     // (It may be stress-tested by specifying StressReflectiveCode.)
4093     // Basically, we want to get into the VM is there's an illegal argument.
4094     Node* bit = intcon(Klass::_lh_instance_slow_path_bit);
4095     initial_slow_test = _gvn.transform( new AndINode(layout_val, bit) );
4096     if (extra_slow_test != intcon(0)) {
4097       initial_slow_test = _gvn.transform( new OrINode(initial_slow_test, extra_slow_test) );
4098     }
4099     // (Macro-expander will further convert this to a Bool, if necessary.)

4110 
4111     // Clear the low bits to extract layout_helper_size_in_bytes:
4112     assert((int)Klass::_lh_instance_slow_path_bit < BytesPerLong, "clear bit");
4113     Node* mask = MakeConX(~ (intptr_t)right_n_bits(LogBytesPerLong));
4114     size = _gvn.transform( new AndXNode(size, mask) );
4115   }
4116   if (return_size_val != nullptr) {
4117     (*return_size_val) = size;
4118   }
4119 
4120   // This is a precise notnull oop of the klass.
4121   // (Actually, it need not be precise if this is a reflective allocation.)
4122   // It's what we cast the result to.
4123   const TypeKlassPtr* tklass = _gvn.type(klass_node)->isa_klassptr();
4124   if (!tklass)  tklass = TypeInstKlassPtr::OBJECT;
4125   const TypeOopPtr* oop_type = tklass->as_instance_type();
4126 
4127   // Now generate allocation code
4128 
4129   // The entire memory state is needed for slow path of the allocation
4130   // since GC and deoptimization can happen.
4131   Node *mem = reset_memory();
4132   set_all_memory(mem); // Create new memory state
4133 
4134   AllocateNode* alloc = new AllocateNode(C, AllocateNode::alloc_type(Type::TOP),
4135                                          control(), mem, i_o(),
4136                                          size, klass_node,
4137                                          initial_slow_test, inline_type_node);
4138 
4139   return set_output_for_allocation(alloc, oop_type, deoptimize_on_exception);
4140 }
4141 
4142 //-------------------------------new_array-------------------------------------
4143 // helper for newarray and anewarray
4144 // The 'length' parameter is (obviously) the length of the array.
4145 // The optional arguments are for specialized use by intrinsics:
4146 //  - If 'return_size_val', report the non-padded array size (sum of header size
4147 //    and array body) to the caller.
4148 //  - deoptimize_on_exception controls how Java exceptions are handled (rethrow vs deoptimize)
4149 Node* GraphKit::new_array(Node* klass_node,     // array klass (maybe variable)
4150                           Node* length,         // number of array elements
4151                           int   nargs,          // number of arguments to push back for uncommon trap
4152                           Node* *return_size_val,
4153                           bool deoptimize_on_exception) {
4154   jint  layout_con = Klass::_lh_neutral_value;
4155   Node* layout_val = get_layout_helper(klass_node, layout_con);
4156   bool  layout_is_con = (layout_val == nullptr);
4157 
4158   if (!layout_is_con && !StressReflectiveCode &&
4159       !too_many_traps(Deoptimization::Reason_class_check)) {
4160     // This is a reflective array creation site.
4161     // Optimistically assume that it is a subtype of Object[],
4162     // so that we can fold up all the address arithmetic.
4163     layout_con = Klass::array_layout_helper(T_OBJECT);
4164     Node* cmp_lh = _gvn.transform( new CmpINode(layout_val, intcon(layout_con)) );
4165     Node* bol_lh = _gvn.transform( new BoolNode(cmp_lh, BoolTest::eq) );
4166     { BuildCutout unless(this, bol_lh, PROB_MAX);
4167       inc_sp(nargs);
4168       uncommon_trap(Deoptimization::Reason_class_check,
4169                     Deoptimization::Action_maybe_recompile);
4170     }
4171     layout_val = nullptr;
4172     layout_is_con = true;
4173   }
4174 
4175   // Generate the initial go-slow test.  Make sure we do not overflow
4176   // if length is huge (near 2Gig) or negative!  We do not need
4177   // exact double-words here, just a close approximation of needed
4178   // double-words.  We can't add any offset or rounding bits, lest we
4179   // take a size -1 of bytes and make it positive.  Use an unsigned
4180   // compare, so negative sizes look hugely positive.
4181   int fast_size_limit = FastAllocateSizeLimit;
4182   if (layout_is_con) {
4183     assert(!StressReflectiveCode, "stress mode does not use these paths");
4184     // Increase the size limit if we have exact knowledge of array type.
4185     int log2_esize = Klass::layout_helper_log2_element_size(layout_con);
4186     fast_size_limit <<= MAX2(LogBytesPerLong - log2_esize, 0);
4187   }
4188 
4189   Node* initial_slow_cmp  = _gvn.transform( new CmpUNode( length, intcon( fast_size_limit ) ) );
4190   Node* initial_slow_test = _gvn.transform( new BoolNode( initial_slow_cmp, BoolTest::gt ) );
4191 
4192   // --- Size Computation ---
4193   // array_size = round_to_heap(array_header + (length << elem_shift));
4194   // where round_to_heap(x) == align_to(x, MinObjAlignmentInBytes)
4195   // and align_to(x, y) == ((x + y-1) & ~(y-1))
4196   // The rounding mask is strength-reduced, if possible.
4197   int round_mask = MinObjAlignmentInBytes - 1;
4198   Node* header_size = nullptr;
4199   // (T_BYTE has the weakest alignment and size restrictions...)
4200   if (layout_is_con) {
4201     int       hsize  = Klass::layout_helper_header_size(layout_con);
4202     int       eshift = Klass::layout_helper_log2_element_size(layout_con);
4203     bool is_flat_array = Klass::layout_helper_is_flatArray(layout_con);
4204     if ((round_mask & ~right_n_bits(eshift)) == 0)
4205       round_mask = 0;  // strength-reduce it if it goes away completely
4206     assert(is_flat_array || (hsize & right_n_bits(eshift)) == 0, "hsize is pre-rounded");
4207     int header_size_min = arrayOopDesc::base_offset_in_bytes(T_BYTE);
4208     assert(header_size_min <= hsize, "generic minimum is smallest");
4209     header_size = intcon(hsize);
4210   } else {
4211     Node* hss   = intcon(Klass::_lh_header_size_shift);
4212     Node* hsm   = intcon(Klass::_lh_header_size_mask);
4213     header_size = _gvn.transform(new URShiftINode(layout_val, hss));
4214     header_size = _gvn.transform(new AndINode(header_size, hsm));
4215   }
4216 
4217   Node* elem_shift = nullptr;
4218   if (layout_is_con) {
4219     int eshift = Klass::layout_helper_log2_element_size(layout_con);
4220     if (eshift != 0)
4221       elem_shift = intcon(eshift);
4222   } else {
4223     // There is no need to mask or shift this value.
4224     // The semantics of LShiftINode include an implicit mask to 0x1F.
4225     assert(Klass::_lh_log2_element_size_shift == 0, "use shift in place");
4226     elem_shift = layout_val;

4273   }
4274   Node* non_rounded_size = _gvn.transform(new AddXNode(headerx, abody));
4275 
4276   if (return_size_val != nullptr) {
4277     // This is the size
4278     (*return_size_val) = non_rounded_size;
4279   }
4280 
4281   Node* size = non_rounded_size;
4282   if (round_mask != 0) {
4283     Node* mask1 = MakeConX(round_mask);
4284     size = _gvn.transform(new AddXNode(size, mask1));
4285     Node* mask2 = MakeConX(~round_mask);
4286     size = _gvn.transform(new AndXNode(size, mask2));
4287   }
4288   // else if round_mask == 0, the size computation is self-rounding
4289 
4290   // Now generate allocation code
4291 
4292   // The entire memory state is needed for slow path of the allocation
4293   // since GC and deoptimization can happen.
4294   Node *mem = reset_memory();
4295   set_all_memory(mem); // Create new memory state
4296 
4297   if (initial_slow_test->is_Bool()) {
4298     // Hide it behind a CMoveI, or else PhaseIdealLoop::split_up will get sick.
4299     initial_slow_test = initial_slow_test->as_Bool()->as_int_value(&_gvn);
4300   }
4301 
4302   const TypeKlassPtr* ary_klass = _gvn.type(klass_node)->isa_klassptr();
4303   const TypeOopPtr* ary_type = ary_klass->as_instance_type();
4304   const TypeAryPtr* ary_ptr = ary_type->isa_aryptr();
4305 
4306   // Inline type array variants:
4307   // - null-ok:              MyValue.ref[] (ciObjArrayKlass "[LMyValue")
4308   // - null-free:            MyValue.val[] (ciObjArrayKlass "[QMyValue")
4309   // - null-free, flat     : MyValue.val[] (ciFlatArrayKlass "[QMyValue")
4310   // Check if array is a null-free, non-flat inline type array
4311   // that needs to be initialized with the default inline type.
4312   Node* default_value = nullptr;
4313   Node* raw_default_value = nullptr;
4314   if (ary_ptr != nullptr && ary_ptr->klass_is_exact()) {
4315     // Array type is known
4316     if (ary_ptr->is_null_free() && !ary_ptr->is_flat()) {
4317       ciInlineKlass* vk = ary_ptr->elem()->inline_klass();
4318       default_value = InlineTypeNode::default_oop(gvn(), vk);
4319     }
4320   } else if (ary_type->can_be_inline_array()) {
4321     // Array type is not known, add runtime checks
4322     assert(!ary_klass->klass_is_exact(), "unexpected exact type");
4323     Node* r = new RegionNode(3);
4324     default_value = new PhiNode(r, TypeInstPtr::BOTTOM);
4325 
4326     Node* bol = array_lh_test(klass_node, Klass::_lh_array_tag_flat_value_bit_inplace | Klass::_lh_null_free_array_bit_inplace, Klass::_lh_null_free_array_bit_inplace);
4327     IfNode* iff = create_and_map_if(control(), bol, PROB_FAIR, COUNT_UNKNOWN);
4328 
4329     // Null-free, non-flat inline type array, initialize with the default value
4330     set_control(_gvn.transform(new IfTrueNode(iff)));
4331     Node* p = basic_plus_adr(klass_node, in_bytes(ArrayKlass::element_klass_offset()));
4332     Node* eklass = _gvn.transform(LoadKlassNode::make(_gvn, control(), immutable_memory(), p, TypeInstPtr::KLASS));
4333     Node* adr_fixed_block_addr = basic_plus_adr(eklass, in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset()));
4334     Node* adr_fixed_block = make_load(control(), adr_fixed_block_addr, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
4335     Node* default_value_offset_addr = basic_plus_adr(adr_fixed_block, in_bytes(InlineKlass::default_value_offset_offset()));
4336     Node* default_value_offset = make_load(control(), default_value_offset_addr, TypeInt::INT, T_INT, MemNode::unordered);
4337     Node* elem_mirror = load_mirror_from_klass(eklass);
4338     Node* default_value_addr = basic_plus_adr(elem_mirror, ConvI2X(default_value_offset));
4339     Node* val = access_load_at(elem_mirror, default_value_addr, TypeInstPtr::MIRROR, TypeInstPtr::NOTNULL, T_OBJECT, IN_HEAP);
4340     r->init_req(1, control());
4341     default_value->init_req(1, val);
4342 
4343     // Otherwise initialize with all zero
4344     r->init_req(2, _gvn.transform(new IfFalseNode(iff)));
4345     default_value->init_req(2, null());
4346 
4347     set_control(_gvn.transform(r));
4348     default_value = _gvn.transform(default_value);
4349   }
4350   if (default_value != nullptr) {
4351     if (UseCompressedOops) {
4352       // With compressed oops, the 64-bit init value is built from two 32-bit compressed oops
4353       default_value = _gvn.transform(new EncodePNode(default_value, default_value->bottom_type()->make_narrowoop()));
4354       Node* lower = _gvn.transform(new CastP2XNode(control(), default_value));
4355       Node* upper = _gvn.transform(new LShiftLNode(lower, intcon(32)));
4356       raw_default_value = _gvn.transform(new OrLNode(lower, upper));
4357     } else {
4358       raw_default_value = _gvn.transform(new CastP2XNode(control(), default_value));
4359     }
4360   }
4361 
4362   Node* valid_length_test = _gvn.intcon(1);
4363   if (ary_type->isa_aryptr()) {
4364     BasicType bt = ary_type->isa_aryptr()->elem()->array_element_basic_type();
4365     jint max = TypeAryPtr::max_array_length(bt);
4366     Node* valid_length_cmp  = _gvn.transform(new CmpUNode(length, intcon(max)));
4367     valid_length_test = _gvn.transform(new BoolNode(valid_length_cmp, BoolTest::le));
4368   }
4369 
4370   // Create the AllocateArrayNode and its result projections
4371   AllocateArrayNode* alloc
4372     = new AllocateArrayNode(C, AllocateArrayNode::alloc_type(TypeInt::INT),
4373                             control(), mem, i_o(),
4374                             size, klass_node,
4375                             initial_slow_test,
4376                             length, valid_length_test,
4377                             default_value, raw_default_value);
4378   // Cast to correct type.  Note that the klass_node may be constant or not,
4379   // and in the latter case the actual array type will be inexact also.
4380   // (This happens via a non-constant argument to inline_native_newArray.)
4381   // In any case, the value of klass_node provides the desired array type.
4382   const TypeInt* length_type = _gvn.find_int_type(length);
4383   if (ary_type->isa_aryptr() && length_type != nullptr) {
4384     // Try to get a better type than POS for the size
4385     ary_type = ary_type->is_aryptr()->cast_to_size(length_type);
4386   }
4387 
4388   Node* javaoop = set_output_for_allocation(alloc, ary_type, deoptimize_on_exception);
4389 
4390   array_ideal_length(alloc, ary_type, true);
4391   return javaoop;
4392 }
4393 
4394 // The following "Ideal_foo" functions are placed here because they recognize
4395 // the graph shapes created by the functions immediately above.
4396 
4397 //---------------------------Ideal_allocation----------------------------------

4504   set_all_memory(ideal.merged_memory());
4505   set_i_o(ideal.i_o());
4506   set_control(ideal.ctrl());
4507 }
4508 
4509 void GraphKit::final_sync(IdealKit& ideal) {
4510   // Final sync IdealKit and graphKit.
4511   sync_kit(ideal);
4512 }
4513 
4514 Node* GraphKit::load_String_length(Node* str, bool set_ctrl) {
4515   Node* len = load_array_length(load_String_value(str, set_ctrl));
4516   Node* coder = load_String_coder(str, set_ctrl);
4517   // Divide length by 2 if coder is UTF16
4518   return _gvn.transform(new RShiftINode(len, coder));
4519 }
4520 
4521 Node* GraphKit::load_String_value(Node* str, bool set_ctrl) {
4522   int value_offset = java_lang_String::value_offset();
4523   const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4524                                                      false, nullptr, Type::Offset(0));
4525   const TypePtr* value_field_type = string_type->add_offset(value_offset);
4526   const TypeAryPtr* value_type = TypeAryPtr::make(TypePtr::NotNull,
4527                                                   TypeAry::make(TypeInt::BYTE, TypeInt::POS, false, false, true, true),
4528                                                   ciTypeArrayKlass::make(T_BYTE), true, Type::Offset(0));
4529   Node* p = basic_plus_adr(str, str, value_offset);
4530   Node* load = access_load_at(str, p, value_field_type, value_type, T_OBJECT,
4531                               IN_HEAP | (set_ctrl ? C2_CONTROL_DEPENDENT_LOAD : 0) | MO_UNORDERED);
4532   return load;
4533 }
4534 
4535 Node* GraphKit::load_String_coder(Node* str, bool set_ctrl) {
4536   if (!CompactStrings) {
4537     return intcon(java_lang_String::CODER_UTF16);
4538   }
4539   int coder_offset = java_lang_String::coder_offset();
4540   const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4541                                                      false, nullptr, Type::Offset(0));
4542   const TypePtr* coder_field_type = string_type->add_offset(coder_offset);
4543 
4544   Node* p = basic_plus_adr(str, str, coder_offset);
4545   Node* load = access_load_at(str, p, coder_field_type, TypeInt::BYTE, T_BYTE,
4546                               IN_HEAP | (set_ctrl ? C2_CONTROL_DEPENDENT_LOAD : 0) | MO_UNORDERED);
4547   return load;
4548 }
4549 
4550 void GraphKit::store_String_value(Node* str, Node* value) {
4551   int value_offset = java_lang_String::value_offset();
4552   const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4553                                                      false, nullptr, Type::Offset(0));
4554   const TypePtr* value_field_type = string_type->add_offset(value_offset);
4555 
4556   access_store_at(str,  basic_plus_adr(str, value_offset), value_field_type,
4557                   value, TypeAryPtr::BYTES, T_OBJECT, IN_HEAP | MO_UNORDERED);
4558 }
4559 
4560 void GraphKit::store_String_coder(Node* str, Node* value) {
4561   int coder_offset = java_lang_String::coder_offset();
4562   const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4563                                                      false, nullptr, Type::Offset(0));
4564   const TypePtr* coder_field_type = string_type->add_offset(coder_offset);
4565 
4566   access_store_at(str, basic_plus_adr(str, coder_offset), coder_field_type,
4567                   value, TypeInt::BYTE, T_BYTE, IN_HEAP | MO_UNORDERED);
4568 }
4569 
4570 // Capture src and dst memory state with a MergeMemNode
4571 Node* GraphKit::capture_memory(const TypePtr* src_type, const TypePtr* dst_type) {
4572   if (src_type == dst_type) {
4573     // Types are equal, we don't need a MergeMemNode
4574     return memory(src_type);
4575   }
4576   MergeMemNode* merge = MergeMemNode::make(map()->memory());
4577   record_for_igvn(merge); // fold it up later, if possible
4578   int src_idx = C->get_alias_index(src_type);
4579   int dst_idx = C->get_alias_index(dst_type);
4580   merge->set_memory_at(src_idx, memory(src_idx));
4581   merge->set_memory_at(dst_idx, memory(dst_idx));
4582   return merge;
4583 }

4656   i_char->init_req(2, AddI(i_char, intcon(2)));
4657 
4658   set_control(IfFalse(iff));
4659   set_memory(st, TypeAryPtr::BYTES);
4660 }
4661 
4662 Node* GraphKit::make_constant_from_field(ciField* field, Node* obj) {
4663   if (!field->is_constant()) {
4664     return nullptr; // Field not marked as constant.
4665   }
4666   ciInstance* holder = nullptr;
4667   if (!field->is_static()) {
4668     ciObject* const_oop = obj->bottom_type()->is_oopptr()->const_oop();
4669     if (const_oop != nullptr && const_oop->is_instance()) {
4670       holder = const_oop->as_instance();
4671     }
4672   }
4673   const Type* con_type = Type::make_constant_from_field(field, holder, field->layout_type(),
4674                                                         /*is_unsigned_load=*/false);
4675   if (con_type != nullptr) {
4676     Node* con = makecon(con_type);
4677     if (field->type()->is_inlinetype()) {
4678       con = InlineTypeNode::make_from_oop(this, con, field->type()->as_inline_klass(), field->is_null_free());
4679     } else if (con_type->is_inlinetypeptr()) {
4680       con = InlineTypeNode::make_from_oop(this, con, con_type->inline_klass(), field->is_null_free());
4681     }
4682     return con;
4683   }
4684   return nullptr;
4685 }
4686 
4687 //---------------------------load_mirror_from_klass----------------------------
4688 // Given a klass oop, load its java mirror (a java.lang.Class oop).
4689 Node* GraphKit::load_mirror_from_klass(Node* klass) {
4690   Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset()));
4691   Node* load = make_load(nullptr, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
4692   // mirror = ((OopHandle)mirror)->resolve();
4693   return access_load(load, TypeInstPtr::MIRROR, T_OBJECT, IN_NATIVE);
4694 }
< prev index next >