6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciUtilities.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "ci/ciNativeEntryPoint.hpp"
29 #include "ci/ciObjArray.hpp"
30 #include "asm/register.hpp"
31 #include "compiler/compileLog.hpp"
32 #include "gc/shared/barrierSet.hpp"
33 #include "gc/shared/c2/barrierSetC2.hpp"
34 #include "interpreter/interpreter.hpp"
35 #include "memory/resourceArea.hpp"
36 #include "opto/addnode.hpp"
37 #include "opto/castnode.hpp"
38 #include "opto/convertnode.hpp"
39 #include "opto/graphKit.hpp"
40 #include "opto/idealKit.hpp"
41 #include "opto/intrinsicnode.hpp"
42 #include "opto/locknode.hpp"
43 #include "opto/machnode.hpp"
44 #include "opto/opaquenode.hpp"
45 #include "opto/parse.hpp"
46 #include "opto/rootnode.hpp"
47 #include "opto/runtime.hpp"
48 #include "opto/subtypenode.hpp"
49 #include "runtime/deoptimization.hpp"
50 #include "runtime/sharedRuntime.hpp"
51 #include "utilities/bitMap.inline.hpp"
52 #include "utilities/powerOfTwo.hpp"
53 #include "utilities/growableArray.hpp"
54
55 //----------------------------GraphKit-----------------------------------------
56 // Main utility constructor.
57 GraphKit::GraphKit(JVMState* jvms)
58 : Phase(Phase::Parser),
59 _env(C->env()),
60 _gvn(*C->initial_gvn()),
61 _barrier_set(BarrierSet::barrier_set()->barrier_set_c2())
62 {
63 _exceptions = jvms->map()->next_exception();
64 if (_exceptions != NULL) jvms->map()->set_next_exception(NULL);
65 set_jvms(jvms);
66 }
67
68 // Private constructor for parser.
69 GraphKit::GraphKit()
70 : Phase(Phase::Parser),
71 _env(C->env()),
72 _gvn(*C->initial_gvn()),
73 _barrier_set(BarrierSet::barrier_set()->barrier_set_c2())
74 {
75 _exceptions = NULL;
76 set_map(NULL);
77 debug_only(_sp = -99);
78 debug_only(set_bci(-99));
79 }
80
81
82
83 //---------------------------clean_stack---------------------------------------
84 // Clear away rubbish from the stack area of the JVM state.
85 // This destroys any arguments that may be waiting on the stack.
813 if (PrintMiscellaneous && (Verbose || WizardMode)) {
814 tty->print_cr("Zombie local %d: ", local);
815 jvms->dump();
816 }
817 return false;
818 }
819 }
820 }
821 return true;
822 }
823
824 #endif //ASSERT
825
826 // Helper function for enforcing certain bytecodes to reexecute if deoptimization happens.
827 static bool should_reexecute_implied_by_bytecode(JVMState *jvms, bool is_anewarray) {
828 ciMethod* cur_method = jvms->method();
829 int cur_bci = jvms->bci();
830 if (cur_method != NULL && cur_bci != InvocationEntryBci) {
831 Bytecodes::Code code = cur_method->java_code_at_bci(cur_bci);
832 return Interpreter::bytecode_should_reexecute(code) ||
833 (is_anewarray && code == Bytecodes::_multianewarray);
834 // Reexecute _multianewarray bytecode which was replaced with
835 // sequence of [a]newarray. See Parse::do_multianewarray().
836 //
837 // Note: interpreter should not have it set since this optimization
838 // is limited by dimensions and guarded by flag so in some cases
839 // multianewarray() runtime calls will be generated and
840 // the bytecode should not be reexecutes (stack will not be reset).
841 } else {
842 return false;
843 }
844 }
845
846 // Helper function for adding JVMState and debug information to node
847 void GraphKit::add_safepoint_edges(SafePointNode* call, bool must_throw) {
848 // Add the safepoint edges to the call (or other safepoint).
849
850 // Make sure dead locals are set to top. This
851 // should help register allocation time and cut down on the size
852 // of the deoptimization information.
853 assert(dead_locals_are_killed(), "garbage in debug info before safepoint");
1073 ciSignature* declared_signature = NULL;
1074 ciMethod* ignored_callee = method()->get_method_at_bci(bci(), ignored_will_link, &declared_signature);
1075 assert(declared_signature != NULL, "cannot be null");
1076 inputs = declared_signature->arg_size_for_bc(code);
1077 int size = declared_signature->return_type()->size();
1078 depth = size - inputs;
1079 }
1080 break;
1081
1082 case Bytecodes::_multianewarray:
1083 {
1084 ciBytecodeStream iter(method());
1085 iter.reset_to_bci(bci());
1086 iter.next();
1087 inputs = iter.get_dimensions();
1088 assert(rsize == 1, "");
1089 depth = rsize - inputs;
1090 }
1091 break;
1092
1093 case Bytecodes::_ireturn:
1094 case Bytecodes::_lreturn:
1095 case Bytecodes::_freturn:
1096 case Bytecodes::_dreturn:
1097 case Bytecodes::_areturn:
1098 assert(rsize == -depth, "");
1099 inputs = rsize;
1100 break;
1101
1102 case Bytecodes::_jsr:
1103 case Bytecodes::_jsr_w:
1104 inputs = 0;
1105 depth = 1; // S.B. depth=1, not zero
1106 break;
1107
1108 default:
1109 // bytecode produces a typed result
1110 inputs = rsize - depth;
1111 assert(inputs >= 0, "");
1112 break;
1155 Node* conv = _gvn.transform( new ConvI2LNode(offset));
1156 Node* mask = _gvn.transform(ConLNode::make((julong) max_juint));
1157 return _gvn.transform( new AndLNode(conv, mask) );
1158 }
1159
1160 Node* GraphKit::ConvL2I(Node* offset) {
1161 // short-circuit a common case
1162 jlong offset_con = find_long_con(offset, (jlong)Type::OffsetBot);
1163 if (offset_con != (jlong)Type::OffsetBot) {
1164 return intcon((int) offset_con);
1165 }
1166 return _gvn.transform( new ConvL2INode(offset));
1167 }
1168
1169 //-------------------------load_object_klass-----------------------------------
1170 Node* GraphKit::load_object_klass(Node* obj) {
1171 // Special-case a fresh allocation to avoid building nodes:
1172 Node* akls = AllocateNode::Ideal_klass(obj, &_gvn);
1173 if (akls != NULL) return akls;
1174 Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
1175 return _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), k_adr, TypeInstPtr::KLASS));
1176 }
1177
1178 //-------------------------load_array_length-----------------------------------
1179 Node* GraphKit::load_array_length(Node* array) {
1180 // Special-case a fresh allocation to avoid building nodes:
1181 AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(array, &_gvn);
1182 Node *alen;
1183 if (alloc == NULL) {
1184 Node *r_adr = basic_plus_adr(array, arrayOopDesc::length_offset_in_bytes());
1185 alen = _gvn.transform( new LoadRangeNode(0, immutable_memory(), r_adr, TypeInt::POS));
1186 } else {
1187 alen = array_ideal_length(alloc, _gvn.type(array)->is_oopptr(), false);
1188 }
1189 return alen;
1190 }
1191
1192 Node* GraphKit::array_ideal_length(AllocateArrayNode* alloc,
1193 const TypeOopPtr* oop_type,
1194 bool replace_length_in_map) {
1195 Node* length = alloc->Ideal_length();
1204 replace_in_map(length, ccast);
1205 }
1206 return ccast;
1207 }
1208 }
1209 return length;
1210 }
1211
1212 //------------------------------do_null_check----------------------------------
1213 // Helper function to do a NULL pointer check. Returned value is
1214 // the incoming address with NULL casted away. You are allowed to use the
1215 // not-null value only if you are control dependent on the test.
1216 #ifndef PRODUCT
1217 extern int explicit_null_checks_inserted,
1218 explicit_null_checks_elided;
1219 #endif
1220 Node* GraphKit::null_check_common(Node* value, BasicType type,
1221 // optional arguments for variations:
1222 bool assert_null,
1223 Node* *null_control,
1224 bool speculative) {
1225 assert(!assert_null || null_control == NULL, "not both at once");
1226 if (stopped()) return top();
1227 NOT_PRODUCT(explicit_null_checks_inserted++);
1228
1229 // Construct NULL check
1230 Node *chk = NULL;
1231 switch(type) {
1232 case T_LONG : chk = new CmpLNode(value, _gvn.zerocon(T_LONG)); break;
1233 case T_INT : chk = new CmpINode(value, _gvn.intcon(0)); break;
1234 case T_ARRAY : // fall through
1235 type = T_OBJECT; // simplify further tests
1236 case T_OBJECT : {
1237 const Type *t = _gvn.type( value );
1238
1239 const TypeOopPtr* tp = t->isa_oopptr();
1240 if (tp != NULL && tp->klass() != NULL && !tp->klass()->is_loaded()
1241 // Only for do_null_check, not any of its siblings:
1242 && !assert_null && null_control == NULL) {
1243 // Usually, any field access or invocation on an unloaded oop type
1244 // will simply fail to link, since the statically linked class is
1245 // likely also to be unloaded. However, in -Xcomp mode, sometimes
1246 // the static class is loaded but the sharper oop type is not.
1247 // Rather than checking for this obscure case in lots of places,
1248 // we simply observe that a null check on an unloaded class
1249 // will always be followed by a nonsense operation, so we
1250 // can just issue the uncommon trap here.
1251 // Our access to the unloaded class will only be correct
1252 // after it has been loaded and initialized, which requires
1253 // a trip through the interpreter.
1311 }
1312 Node *oldcontrol = control();
1313 set_control(cfg);
1314 Node *res = cast_not_null(value);
1315 set_control(oldcontrol);
1316 NOT_PRODUCT(explicit_null_checks_elided++);
1317 return res;
1318 }
1319 cfg = IfNode::up_one_dom(cfg, /*linear_only=*/ true);
1320 if (cfg == NULL) break; // Quit at region nodes
1321 depth++;
1322 }
1323 }
1324
1325 //-----------
1326 // Branch to failure if null
1327 float ok_prob = PROB_MAX; // a priori estimate: nulls never happen
1328 Deoptimization::DeoptReason reason;
1329 if (assert_null) {
1330 reason = Deoptimization::reason_null_assert(speculative);
1331 } else if (type == T_OBJECT) {
1332 reason = Deoptimization::reason_null_check(speculative);
1333 } else {
1334 reason = Deoptimization::Reason_div0_check;
1335 }
1336 // %%% Since Reason_unhandled is not recorded on a per-bytecode basis,
1337 // ciMethodData::has_trap_at will return a conservative -1 if any
1338 // must-be-null assertion has failed. This could cause performance
1339 // problems for a method after its first do_null_assert failure.
1340 // Consider using 'Reason_class_check' instead?
1341
1342 // To cause an implicit null check, we set the not-null probability
1343 // to the maximum (PROB_MAX). For an explicit check the probability
1344 // is set to a smaller value.
1345 if (null_control != NULL || too_many_traps(reason)) {
1346 // probability is less likely
1347 ok_prob = PROB_LIKELY_MAG(3);
1348 } else if (!assert_null &&
1349 (ImplicitNullCheckThreshold > 0) &&
1350 method() != NULL &&
1351 (method()->method_data()->trap_count(reason)
1385 }
1386
1387 if (assert_null) {
1388 // Cast obj to null on this path.
1389 replace_in_map(value, zerocon(type));
1390 return zerocon(type);
1391 }
1392
1393 // Cast obj to not-null on this path, if there is no null_control.
1394 // (If there is a null_control, a non-null value may come back to haunt us.)
1395 if (type == T_OBJECT) {
1396 Node* cast = cast_not_null(value, false);
1397 if (null_control == NULL || (*null_control) == top())
1398 replace_in_map(value, cast);
1399 value = cast;
1400 }
1401
1402 return value;
1403 }
1404
1405
1406 //------------------------------cast_not_null----------------------------------
1407 // Cast obj to not-null on this path
1408 Node* GraphKit::cast_not_null(Node* obj, bool do_replace_in_map) {
1409 const Type *t = _gvn.type(obj);
1410 const Type *t_not_null = t->join_speculative(TypePtr::NOTNULL);
1411 // Object is already not-null?
1412 if( t == t_not_null ) return obj;
1413
1414 Node *cast = new CastPPNode(obj,t_not_null);
1415 cast->init_req(0, control());
1416 cast = _gvn.transform( cast );
1417
1418 // Scan for instances of 'obj' in the current JVM mapping.
1419 // These instances are known to be not-null after the test.
1420 if (do_replace_in_map)
1421 replace_in_map(obj, cast);
1422
1423 return cast; // Return casted value
1424 }
1425
1426 // Sometimes in intrinsics, we implicitly know an object is not null
1427 // (there's no actual null check) so we can cast it to not null. In
1428 // the course of optimizations, the input to the cast can become null.
1522 MemNode::MemOrd mo,
1523 LoadNode::ControlDependency control_dependency,
1524 bool require_atomic_access,
1525 bool unaligned,
1526 bool mismatched,
1527 bool unsafe,
1528 uint8_t barrier_data) {
1529 assert(adr_idx != Compile::AliasIdxTop, "use other make_load factory" );
1530 const TypePtr* adr_type = NULL; // debug-mode-only argument
1531 debug_only(adr_type = C->get_adr_type(adr_idx));
1532 Node* mem = memory(adr_idx);
1533 Node* ld;
1534 if (require_atomic_access && bt == T_LONG) {
1535 ld = LoadLNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data);
1536 } else if (require_atomic_access && bt == T_DOUBLE) {
1537 ld = LoadDNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data);
1538 } else {
1539 ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data);
1540 }
1541 ld = _gvn.transform(ld);
1542 if (((bt == T_OBJECT) && C->do_escape_analysis()) || C->eliminate_boxing()) {
1543 // Improve graph before escape analysis and boxing elimination.
1544 record_for_igvn(ld);
1545 }
1546 return ld;
1547 }
1548
1549 Node* GraphKit::store_to_memory(Node* ctl, Node* adr, Node *val, BasicType bt,
1550 int adr_idx,
1551 MemNode::MemOrd mo,
1552 bool require_atomic_access,
1553 bool unaligned,
1554 bool mismatched,
1555 bool unsafe) {
1556 assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
1557 const TypePtr* adr_type = NULL;
1558 debug_only(adr_type = C->get_adr_type(adr_idx));
1559 Node *mem = memory(adr_idx);
1560 Node* st;
1561 if (require_atomic_access && bt == T_LONG) {
1562 st = StoreLNode::make_atomic(ctl, mem, adr, adr_type, val, mo);
1573 }
1574 if (unsafe) {
1575 st->as_Store()->set_unsafe_access();
1576 }
1577 st = _gvn.transform(st);
1578 set_memory(st, adr_idx);
1579 // Back-to-back stores can only remove intermediate store with DU info
1580 // so push on worklist for optimizer.
1581 if (mem->req() > MemNode::Address && adr == mem->in(MemNode::Address))
1582 record_for_igvn(st);
1583
1584 return st;
1585 }
1586
1587 Node* GraphKit::access_store_at(Node* obj,
1588 Node* adr,
1589 const TypePtr* adr_type,
1590 Node* val,
1591 const Type* val_type,
1592 BasicType bt,
1593 DecoratorSet decorators) {
1594 // Transformation of a value which could be NULL pointer (CastPP #NULL)
1595 // could be delayed during Parse (for example, in adjust_map_after_if()).
1596 // Execute transformation here to avoid barrier generation in such case.
1597 if (_gvn.type(val) == TypePtr::NULL_PTR) {
1598 val = _gvn.makecon(TypePtr::NULL_PTR);
1599 }
1600
1601 if (stopped()) {
1602 return top(); // Dead path ?
1603 }
1604
1605 assert(val != NULL, "not dead path");
1606
1607 C2AccessValuePtr addr(adr, adr_type);
1608 C2AccessValue value(val, val_type);
1609 C2ParseAccess access(this, decorators | C2_WRITE_ACCESS, bt, obj, addr);
1610 if (access.is_raw()) {
1611 return _barrier_set->BarrierSetC2::store_at(access, value);
1612 } else {
1613 return _barrier_set->store_at(access, value);
1614 }
1615 }
1616
1617 Node* GraphKit::access_load_at(Node* obj, // containing obj
1618 Node* adr, // actual adress to store val at
1619 const TypePtr* adr_type,
1620 const Type* val_type,
1621 BasicType bt,
1622 DecoratorSet decorators) {
1623 if (stopped()) {
1624 return top(); // Dead path ?
1625 }
1626
1627 C2AccessValuePtr addr(adr, adr_type);
1628 C2ParseAccess access(this, decorators | C2_READ_ACCESS, bt, obj, addr);
1629 if (access.is_raw()) {
1630 return _barrier_set->BarrierSetC2::load_at(access, val_type);
1631 } else {
1632 return _barrier_set->load_at(access, val_type);
1633 }
1634 }
1635
1636 Node* GraphKit::access_load(Node* adr, // actual adress to load val at
1637 const Type* val_type,
1638 BasicType bt,
1639 DecoratorSet decorators) {
1640 if (stopped()) {
1641 return top(); // Dead path ?
1642 }
1643
1644 C2AccessValuePtr addr(adr, adr->bottom_type()->is_ptr());
1645 C2ParseAccess access(this, decorators | C2_READ_ACCESS, bt, NULL, addr);
1646 if (access.is_raw()) {
1647 return _barrier_set->BarrierSetC2::load_at(access, val_type);
1648 } else {
1714 const Type* value_type,
1715 BasicType bt,
1716 DecoratorSet decorators) {
1717 C2AccessValuePtr addr(adr, adr_type);
1718 C2AtomicParseAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, bt, obj, addr, alias_idx);
1719 if (access.is_raw()) {
1720 return _barrier_set->BarrierSetC2::atomic_add_at(access, new_val, value_type);
1721 } else {
1722 return _barrier_set->atomic_add_at(access, new_val, value_type);
1723 }
1724 }
1725
1726 void GraphKit::access_clone(Node* src, Node* dst, Node* size, bool is_array) {
1727 return _barrier_set->clone(this, src, dst, size, is_array);
1728 }
1729
1730 //-------------------------array_element_address-------------------------
1731 Node* GraphKit::array_element_address(Node* ary, Node* idx, BasicType elembt,
1732 const TypeInt* sizetype, Node* ctrl) {
1733 uint shift = exact_log2(type2aelembytes(elembt));
1734 uint header = arrayOopDesc::base_offset_in_bytes(elembt);
1735
1736 // short-circuit a common case (saves lots of confusing waste motion)
1737 jint idx_con = find_int_con(idx, -1);
1738 if (idx_con >= 0) {
1739 intptr_t offset = header + ((intptr_t)idx_con << shift);
1740 return basic_plus_adr(ary, offset);
1741 }
1742
1743 // must be correct type for alignment purposes
1744 Node* base = basic_plus_adr(ary, header);
1745 idx = Compile::conv_I2X_index(&_gvn, idx, sizetype, ctrl);
1746 Node* scale = _gvn.transform( new LShiftXNode(idx, intcon(shift)) );
1747 return basic_plus_adr(ary, base, scale);
1748 }
1749
1750 //-------------------------load_array_element-------------------------
1751 Node* GraphKit::load_array_element(Node* ary, Node* idx, const TypeAryPtr* arytype, bool set_ctrl) {
1752 const Type* elemtype = arytype->elem();
1753 BasicType elembt = elemtype->array_element_basic_type();
1754 Node* adr = array_element_address(ary, idx, elembt, arytype->size());
1755 if (elembt == T_NARROWOOP) {
1756 elembt = T_OBJECT; // To satisfy switch in LoadNode::make()
1757 }
1758 Node* ld = access_load_at(ary, adr, arytype, elemtype, elembt,
1759 IN_HEAP | IS_ARRAY | (set_ctrl ? C2_CONTROL_DEPENDENT_LOAD : 0));
1760 return ld;
1761 }
1762
1763 //-------------------------set_arguments_for_java_call-------------------------
1764 // Arguments (pre-popped from the stack) are taken from the JVMS.
1765 void GraphKit::set_arguments_for_java_call(CallJavaNode* call) {
1766 // Add the call arguments:
1767 uint nargs = call->method()->arg_size();
1768 for (uint i = 0; i < nargs; i++) {
1769 Node* arg = argument(i);
1770 call->init_req(i + TypeFunc::Parms, arg);
1771 }
1772 }
1773
1774 //---------------------------set_edges_for_java_call---------------------------
1775 // Connect a newly created call into the current JVMS.
1776 // A return value node (if any) is returned from set_edges_for_java_call.
1777 void GraphKit::set_edges_for_java_call(CallJavaNode* call, bool must_throw, bool separate_io_proj) {
1778
1779 // Add the predefined inputs:
1780 call->init_req( TypeFunc::Control, control() );
1781 call->init_req( TypeFunc::I_O , i_o() );
1782 call->init_req( TypeFunc::Memory , reset_memory() );
1783 call->init_req( TypeFunc::FramePtr, frameptr() );
1784 call->init_req( TypeFunc::ReturnAdr, top() );
1785
1786 add_safepoint_edges(call, must_throw);
1787
1788 Node* xcall = _gvn.transform(call);
1789
1790 if (xcall == top()) {
1791 set_control(top());
1792 return;
1793 }
1794 assert(xcall == call, "call identity is stable");
1795
1796 // Re-use the current map to produce the result.
1797
1798 set_control(_gvn.transform(new ProjNode(call, TypeFunc::Control)));
1799 set_i_o( _gvn.transform(new ProjNode(call, TypeFunc::I_O , separate_io_proj)));
1800 set_all_memory_call(xcall, separate_io_proj);
1801
1802 //return xcall; // no need, caller already has it
1803 }
1804
1805 Node* GraphKit::set_results_for_java_call(CallJavaNode* call, bool separate_io_proj, bool deoptimize) {
1806 if (stopped()) return top(); // maybe the call folded up?
1807
1808 // Capture the return value, if any.
1809 Node* ret;
1810 if (call->method() == NULL ||
1811 call->method()->return_type()->basic_type() == T_VOID)
1812 ret = top();
1813 else ret = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
1814
1815 // Note: Since any out-of-line call can produce an exception,
1816 // we always insert an I_O projection from the call into the result.
1817
1818 make_slow_call_ex(call, env()->Throwable_klass(), separate_io_proj, deoptimize);
1819
1820 if (separate_io_proj) {
1821 // The caller requested separate projections be used by the fall
1822 // through and exceptional paths, so replace the projections for
1823 // the fall through path.
1824 set_i_o(_gvn.transform( new ProjNode(call, TypeFunc::I_O) ));
1825 set_all_memory(_gvn.transform( new ProjNode(call, TypeFunc::Memory) ));
1826 }
1827 return ret;
1828 }
1829
1830 //--------------------set_predefined_input_for_runtime_call--------------------
1831 // Reading and setting the memory state is way conservative here.
1832 // The real problem is that I am not doing real Type analysis on memory,
1833 // so I cannot distinguish card mark stores from other stores. Across a GC
1834 // point the Store Barrier and the card mark memory has to agree. I cannot
1835 // have a card mark store and its barrier split across the GC point from
1836 // either above or below. Here I get that to happen by reading ALL of memory.
1837 // A better answer would be to separate out card marks from other memory.
1838 // For now, return the input memory state, so that it can be reused
1839 // after the call, if this call has restricted memory effects.
1840 Node* GraphKit::set_predefined_input_for_runtime_call(SafePointNode* call, Node* narrow_mem) {
1841 // Set fixed predefined input arguments
1842 Node* memory = reset_memory();
1843 Node* m = narrow_mem == NULL ? memory : narrow_mem;
1844 call->init_req( TypeFunc::Control, control() );
1845 call->init_req( TypeFunc::I_O, top() ); // does no i/o
1846 call->init_req( TypeFunc::Memory, m ); // may gc ptrs
1897 if (use->is_MergeMem()) {
1898 wl.push(use);
1899 }
1900 }
1901 }
1902
1903 // Replace the call with the current state of the kit.
1904 void GraphKit::replace_call(CallNode* call, Node* result, bool do_replaced_nodes) {
1905 JVMState* ejvms = NULL;
1906 if (has_exceptions()) {
1907 ejvms = transfer_exceptions_into_jvms();
1908 }
1909
1910 ReplacedNodes replaced_nodes = map()->replaced_nodes();
1911 ReplacedNodes replaced_nodes_exception;
1912 Node* ex_ctl = top();
1913
1914 SafePointNode* final_state = stop();
1915
1916 // Find all the needed outputs of this call
1917 CallProjections callprojs;
1918 call->extract_projections(&callprojs, true);
1919
1920 Unique_Node_List wl;
1921 Node* init_mem = call->in(TypeFunc::Memory);
1922 Node* final_mem = final_state->in(TypeFunc::Memory);
1923 Node* final_ctl = final_state->in(TypeFunc::Control);
1924 Node* final_io = final_state->in(TypeFunc::I_O);
1925
1926 // Replace all the old call edges with the edges from the inlining result
1927 if (callprojs.fallthrough_catchproj != NULL) {
1928 C->gvn_replace_by(callprojs.fallthrough_catchproj, final_ctl);
1929 }
1930 if (callprojs.fallthrough_memproj != NULL) {
1931 if (final_mem->is_MergeMem()) {
1932 // Parser's exits MergeMem was not transformed but may be optimized
1933 final_mem = _gvn.transform(final_mem);
1934 }
1935 C->gvn_replace_by(callprojs.fallthrough_memproj, final_mem);
1936 add_mergemem_users_to_worklist(wl, final_mem);
1937 }
1938 if (callprojs.fallthrough_ioproj != NULL) {
1939 C->gvn_replace_by(callprojs.fallthrough_ioproj, final_io);
1940 }
1941
1942 // Replace the result with the new result if it exists and is used
1943 if (callprojs.resproj != NULL && result != NULL) {
1944 C->gvn_replace_by(callprojs.resproj, result);
1945 }
1946
1947 if (ejvms == NULL) {
1948 // No exception edges to simply kill off those paths
1949 if (callprojs.catchall_catchproj != NULL) {
1950 C->gvn_replace_by(callprojs.catchall_catchproj, C->top());
1951 }
1952 if (callprojs.catchall_memproj != NULL) {
1953 C->gvn_replace_by(callprojs.catchall_memproj, C->top());
1954 }
1955 if (callprojs.catchall_ioproj != NULL) {
1956 C->gvn_replace_by(callprojs.catchall_ioproj, C->top());
1957 }
1958 // Replace the old exception object with top
1959 if (callprojs.exobj != NULL) {
1960 C->gvn_replace_by(callprojs.exobj, C->top());
1961 }
1962 } else {
1963 GraphKit ekit(ejvms);
1964
1965 // Load my combined exception state into the kit, with all phis transformed:
1966 SafePointNode* ex_map = ekit.combine_and_pop_all_exception_states();
1967 replaced_nodes_exception = ex_map->replaced_nodes();
1968
1969 Node* ex_oop = ekit.use_exception_state(ex_map);
1970
1971 if (callprojs.catchall_catchproj != NULL) {
1972 C->gvn_replace_by(callprojs.catchall_catchproj, ekit.control());
1973 ex_ctl = ekit.control();
1974 }
1975 if (callprojs.catchall_memproj != NULL) {
1976 Node* ex_mem = ekit.reset_memory();
1977 C->gvn_replace_by(callprojs.catchall_memproj, ex_mem);
1978 add_mergemem_users_to_worklist(wl, ex_mem);
1979 }
1980 if (callprojs.catchall_ioproj != NULL) {
1981 C->gvn_replace_by(callprojs.catchall_ioproj, ekit.i_o());
1982 }
1983
1984 // Replace the old exception object with the newly created one
1985 if (callprojs.exobj != NULL) {
1986 C->gvn_replace_by(callprojs.exobj, ex_oop);
1987 }
1988 }
1989
1990 // Disconnect the call from the graph
1991 call->disconnect_inputs(C);
1992 C->gvn_replace_by(call, C->top());
1993
1994 // Clean up any MergeMems that feed other MergeMems since the
1995 // optimizer doesn't like that.
1996 while (wl.size() > 0) {
1997 _gvn.transform(wl.pop());
1998 }
1999
2000 if (callprojs.fallthrough_catchproj != NULL && !final_ctl->is_top() && do_replaced_nodes) {
2001 replaced_nodes.apply(C, final_ctl);
2002 }
2003 if (!ex_ctl->is_top() && do_replaced_nodes) {
2004 replaced_nodes_exception.apply(C, ex_ctl);
2005 }
2006 }
2007
2008
2009 //------------------------------increment_counter------------------------------
2010 // for statistics: increment a VM counter by 1
2011
2012 void GraphKit::increment_counter(address counter_addr) {
2013 Node* adr1 = makecon(TypeRawPtr::make(counter_addr));
2014 increment_counter(adr1);
2015 }
2016
2017 void GraphKit::increment_counter(Node* counter_addr) {
2018 int adr_type = Compile::AliasIdxRaw;
2019 Node* ctrl = control();
2020 Node* cnt = make_load(ctrl, counter_addr, TypeLong::LONG, T_LONG, adr_type, MemNode::unordered);
2178 *
2179 * @param n node that the type applies to
2180 * @param exact_kls type from profiling
2181 * @param maybe_null did profiling see null?
2182 *
2183 * @return node with improved type
2184 */
2185 Node* GraphKit::record_profile_for_speculation(Node* n, ciKlass* exact_kls, ProfilePtrKind ptr_kind) {
2186 const Type* current_type = _gvn.type(n);
2187 assert(UseTypeSpeculation, "type speculation must be on");
2188
2189 const TypePtr* speculative = current_type->speculative();
2190
2191 // Should the klass from the profile be recorded in the speculative type?
2192 if (current_type->would_improve_type(exact_kls, jvms()->depth())) {
2193 const TypeKlassPtr* tklass = TypeKlassPtr::make(exact_kls);
2194 const TypeOopPtr* xtype = tklass->as_instance_type();
2195 assert(xtype->klass_is_exact(), "Should be exact");
2196 // Any reason to believe n is not null (from this profiling or a previous one)?
2197 assert(ptr_kind != ProfileAlwaysNull, "impossible here");
2198 const TypePtr* ptr = (ptr_kind == ProfileMaybeNull && current_type->speculative_maybe_null()) ? TypePtr::BOTTOM : TypePtr::NOTNULL;
2199 // record the new speculative type's depth
2200 speculative = xtype->cast_to_ptr_type(ptr->ptr())->is_ptr();
2201 speculative = speculative->with_inline_depth(jvms()->depth());
2202 } else if (current_type->would_improve_ptr(ptr_kind)) {
2203 // Profiling report that null was never seen so we can change the
2204 // speculative type to non null ptr.
2205 if (ptr_kind == ProfileAlwaysNull) {
2206 speculative = TypePtr::NULL_PTR;
2207 } else {
2208 assert(ptr_kind == ProfileNeverNull, "nothing else is an improvement");
2209 const TypePtr* ptr = TypePtr::NOTNULL;
2210 if (speculative != NULL) {
2211 speculative = speculative->cast_to_ptr_type(ptr->ptr())->is_ptr();
2212 } else {
2213 speculative = ptr;
2214 }
2215 }
2216 }
2217
2218 if (speculative != current_type->speculative()) {
2219 // Build a type with a speculative type (what we think we know
2220 // about the type but will need a guard when we use it)
2221 const TypeOopPtr* spec_type = TypeOopPtr::make(TypePtr::BotPTR, Type::OffsetBot, TypeOopPtr::InstanceBot, speculative);
2222 // We're changing the type, we need a new CheckCast node to carry
2223 // the new type. The new type depends on the control: what
2224 // profiling tells us is only valid from here as far as we can
2225 // tell.
2226 Node* cast = new CheckCastPPNode(control(), n, current_type->remove_speculative()->join_speculative(spec_type));
2227 cast = _gvn.transform(cast);
2228 replace_in_map(n, cast);
2229 n = cast;
2230 }
2231
2232 return n;
2233 }
2234
2235 /**
2236 * Record profiling data from receiver profiling at an invoke with the
2237 * type system so that it can propagate it (speculation)
2238 *
2239 * @param n receiver node
2240 *
2241 * @return node with improved type
2242 */
2243 Node* GraphKit::record_profiled_receiver_for_speculation(Node* n) {
2244 if (!UseTypeSpeculation) {
2245 return n;
2246 }
2247 ciKlass* exact_kls = profile_has_unique_klass();
2248 ProfilePtrKind ptr_kind = ProfileMaybeNull;
2249 if ((java_bc() == Bytecodes::_checkcast ||
2250 java_bc() == Bytecodes::_instanceof ||
2251 java_bc() == Bytecodes::_aastore) &&
2252 method()->method_data()->is_mature()) {
2253 ciProfileData* data = method()->method_data()->bci_to_data(bci());
2254 if (data != NULL) {
2255 if (!data->as_BitData()->null_seen()) {
2256 ptr_kind = ProfileNeverNull;
2257 } else {
2258 assert(data->is_ReceiverTypeData(), "bad profile data type");
2259 ciReceiverTypeData* call = (ciReceiverTypeData*)data->as_ReceiverTypeData();
2260 uint i = 0;
2261 for (; i < call->row_limit(); i++) {
2262 ciKlass* receiver = call->receiver(i);
2263 if (receiver != NULL) {
2264 break;
2265 }
2266 }
2267 ptr_kind = (i == call->row_limit()) ? ProfileAlwaysNull : ProfileMaybeNull;
2268 }
2269 }
2270 }
2271 return record_profile_for_speculation(n, exact_kls, ptr_kind);
2272 }
2273
2274 /**
2275 * Record profiling data from argument profiling at an invoke with the
2276 * type system so that it can propagate it (speculation)
2277 *
2278 * @param dest_method target method for the call
2279 * @param bc what invoke bytecode is this?
2280 */
2281 void GraphKit::record_profiled_arguments_for_speculation(ciMethod* dest_method, Bytecodes::Code bc) {
2282 if (!UseTypeSpeculation) {
2283 return;
2284 }
2285 const TypeFunc* tf = TypeFunc::make(dest_method);
2286 int nargs = tf->domain()->cnt() - TypeFunc::Parms;
2287 int skip = Bytecodes::has_receiver(bc) ? 1 : 0;
2288 for (int j = skip, i = 0; j < nargs && i < TypeProfileArgsLimit; j++) {
2289 const Type *targ = tf->domain()->field_at(j + TypeFunc::Parms);
2290 if (is_reference_type(targ->basic_type())) {
2291 ProfilePtrKind ptr_kind = ProfileMaybeNull;
2292 ciKlass* better_type = NULL;
2293 if (method()->argument_profiled_type(bci(), i, better_type, ptr_kind)) {
2294 record_profile_for_speculation(argument(j), better_type, ptr_kind);
2295 }
2296 i++;
2297 }
2298 }
2299 }
2300
2301 /**
2302 * Record profiling data from parameter profiling at an invoke with
2303 * the type system so that it can propagate it (speculation)
2304 */
2305 void GraphKit::record_profiled_parameters_for_speculation() {
2306 if (!UseTypeSpeculation) {
2307 return;
2308 }
2309 for (int i = 0, j = 0; i < method()->arg_size() ; i++) {
2323 * the type system so that it can propagate it (speculation)
2324 */
2325 void GraphKit::record_profiled_return_for_speculation() {
2326 if (!UseTypeSpeculation) {
2327 return;
2328 }
2329 ProfilePtrKind ptr_kind = ProfileMaybeNull;
2330 ciKlass* better_type = NULL;
2331 if (method()->return_profiled_type(bci(), better_type, ptr_kind)) {
2332 // If profiling reports a single type for the return value,
2333 // feed it to the type system so it can propagate it as a
2334 // speculative type
2335 record_profile_for_speculation(stack(sp()-1), better_type, ptr_kind);
2336 }
2337 }
2338
2339 void GraphKit::round_double_arguments(ciMethod* dest_method) {
2340 if (Matcher::strict_fp_requires_explicit_rounding) {
2341 // (Note: TypeFunc::make has a cache that makes this fast.)
2342 const TypeFunc* tf = TypeFunc::make(dest_method);
2343 int nargs = tf->domain()->cnt() - TypeFunc::Parms;
2344 for (int j = 0; j < nargs; j++) {
2345 const Type *targ = tf->domain()->field_at(j + TypeFunc::Parms);
2346 if (targ->basic_type() == T_DOUBLE) {
2347 // If any parameters are doubles, they must be rounded before
2348 // the call, dstore_rounding does gvn.transform
2349 Node *arg = argument(j);
2350 arg = dstore_rounding(arg);
2351 set_argument(j, arg);
2352 }
2353 }
2354 }
2355 }
2356
2357 // rounding for strict float precision conformance
2358 Node* GraphKit::precision_rounding(Node* n) {
2359 if (Matcher::strict_fp_requires_explicit_rounding) {
2360 #ifdef IA32
2361 if (UseSSE == 0) {
2362 return _gvn.transform(new RoundFloatNode(0, n));
2363 }
2364 #else
2365 Unimplemented();
2489 Node* parm0, Node* parm1,
2490 Node* parm2, Node* parm3,
2491 Node* parm4, Node* parm5,
2492 Node* parm6, Node* parm7,
2493 Node* parm8) {
2494 assert(call_addr != NULL, "must not call NULL targets");
2495
2496 // Slow-path call
2497 bool is_leaf = !(flags & RC_NO_LEAF);
2498 bool has_io = (!is_leaf && !(flags & RC_NO_IO));
2499 if (call_name == NULL) {
2500 assert(!is_leaf, "must supply name for leaf");
2501 call_name = OptoRuntime::stub_name(call_addr);
2502 }
2503 CallNode* call;
2504 if (!is_leaf) {
2505 call = new CallStaticJavaNode(call_type, call_addr, call_name, adr_type);
2506 } else if (flags & RC_NO_FP) {
2507 call = new CallLeafNoFPNode(call_type, call_addr, call_name, adr_type);
2508 } else if (flags & RC_VECTOR){
2509 uint num_bits = call_type->range()->field_at(TypeFunc::Parms)->is_vect()->length_in_bytes() * BitsPerByte;
2510 call = new CallLeafVectorNode(call_type, call_addr, call_name, adr_type, num_bits);
2511 } else {
2512 call = new CallLeafNode(call_type, call_addr, call_name, adr_type);
2513 }
2514
2515 // The following is similar to set_edges_for_java_call,
2516 // except that the memory effects of the call are restricted to AliasIdxRaw.
2517
2518 // Slow path call has no side-effects, uses few values
2519 bool wide_in = !(flags & RC_NARROW_MEM);
2520 bool wide_out = (C->get_alias_index(adr_type) == Compile::AliasIdxBot);
2521
2522 Node* prev_mem = NULL;
2523 if (wide_in) {
2524 prev_mem = set_predefined_input_for_runtime_call(call);
2525 } else {
2526 assert(!wide_out, "narrow in => narrow out");
2527 Node* narrow_mem = memory(adr_type);
2528 prev_mem = set_predefined_input_for_runtime_call(call, narrow_mem);
2529 }
2589
2590 //-----------------------------make_native_call-------------------------------
2591 Node* GraphKit::make_native_call(address call_addr, const TypeFunc* call_type, uint nargs, ciNativeEntryPoint* nep) {
2592 // Select just the actual call args to pass on
2593 // [MethodHandle fallback, long addr, HALF addr, ... args , NativeEntryPoint nep]
2594 // | |
2595 // V V
2596 // [ ... args ]
2597 uint n_filtered_args = nargs - 4; // -fallback, -addr (2), -nep;
2598 ResourceMark rm;
2599 Node** argument_nodes = NEW_RESOURCE_ARRAY(Node*, n_filtered_args);
2600 const Type** arg_types = TypeTuple::fields(n_filtered_args);
2601 GrowableArray<VMReg> arg_regs(C->comp_arena(), n_filtered_args, n_filtered_args, VMRegImpl::Bad());
2602
2603 VMReg* argRegs = nep->argMoves();
2604 {
2605 for (uint vm_arg_pos = 0, java_arg_read_pos = 0;
2606 vm_arg_pos < n_filtered_args; vm_arg_pos++) {
2607 uint vm_unfiltered_arg_pos = vm_arg_pos + 3; // +3 to skip fallback handle argument and addr (2 since long)
2608 Node* node = argument(vm_unfiltered_arg_pos);
2609 const Type* type = call_type->domain()->field_at(TypeFunc::Parms + vm_unfiltered_arg_pos);
2610 VMReg reg = type == Type::HALF
2611 ? VMRegImpl::Bad()
2612 : argRegs[java_arg_read_pos++];
2613
2614 argument_nodes[vm_arg_pos] = node;
2615 arg_types[TypeFunc::Parms + vm_arg_pos] = type;
2616 arg_regs.at_put(vm_arg_pos, reg);
2617 }
2618 }
2619
2620 uint n_returns = call_type->range()->cnt() - TypeFunc::Parms;
2621 GrowableArray<VMReg> ret_regs(C->comp_arena(), n_returns, n_returns, VMRegImpl::Bad());
2622 const Type** ret_types = TypeTuple::fields(n_returns);
2623
2624 VMReg* retRegs = nep->returnMoves();
2625 {
2626 for (uint vm_ret_pos = 0, java_ret_read_pos = 0;
2627 vm_ret_pos < n_returns; vm_ret_pos++) { // 0 or 1
2628 const Type* type = call_type->range()->field_at(TypeFunc::Parms + vm_ret_pos);
2629 VMReg reg = type == Type::HALF
2630 ? VMRegImpl::Bad()
2631 : retRegs[java_ret_read_pos++];
2632
2633 ret_regs.at_put(vm_ret_pos, reg);
2634 ret_types[TypeFunc::Parms + vm_ret_pos] = type;
2635 }
2636 }
2637
2638 const TypeFunc* new_call_type = TypeFunc::make(
2639 TypeTuple::make(TypeFunc::Parms + n_filtered_args, arg_types),
2640 TypeTuple::make(TypeFunc::Parms + n_returns, ret_types)
2641 );
2642
2643 if (nep->need_transition()) {
2644 RuntimeStub* invoker = SharedRuntime::make_native_invoker(call_addr,
2645 nep->shadow_space(),
2646 arg_regs, ret_regs);
2647 if (invoker == NULL) {
2648 C->record_failure("native invoker not implemented on this platform");
2935
2936 // Now do a linear scan of the secondary super-klass array. Again, no real
2937 // performance impact (too rare) but it's gotta be done.
2938 // Since the code is rarely used, there is no penalty for moving it
2939 // out of line, and it can only improve I-cache density.
2940 // The decision to inline or out-of-line this final check is platform
2941 // dependent, and is found in the AD file definition of PartialSubtypeCheck.
2942 Node* psc = gvn.transform(
2943 new PartialSubtypeCheckNode(*ctrl, subklass, superklass));
2944
2945 IfNode *iff4 = gen_subtype_check_compare(*ctrl, psc, gvn.zerocon(T_OBJECT), BoolTest::ne, PROB_FAIR, gvn, T_ADDRESS);
2946 r_not_subtype->init_req(2, gvn.transform(new IfTrueNode (iff4)));
2947 r_ok_subtype ->init_req(3, gvn.transform(new IfFalseNode(iff4)));
2948
2949 // Return false path; set default control to true path.
2950 *ctrl = gvn.transform(r_ok_subtype);
2951 return gvn.transform(r_not_subtype);
2952 }
2953
2954 Node* GraphKit::gen_subtype_check(Node* obj_or_subklass, Node* superklass) {
2955 bool expand_subtype_check = C->post_loop_opts_phase() || // macro node expansion is over
2956 ExpandSubTypeCheckAtParseTime; // forced expansion
2957 if (expand_subtype_check) {
2958 MergeMemNode* mem = merged_memory();
2959 Node* ctrl = control();
2960 Node* subklass = obj_or_subklass;
2961 if (!_gvn.type(obj_or_subklass)->isa_klassptr()) {
2962 subklass = load_object_klass(obj_or_subklass);
2963 }
2964
2965 Node* n = Phase::gen_subtype_check(subklass, superklass, &ctrl, mem, _gvn);
2966 set_control(ctrl);
2967 return n;
2968 }
2969
2970 Node* check = _gvn.transform(new SubTypeCheckNode(C, obj_or_subklass, superklass));
2971 Node* bol = _gvn.transform(new BoolNode(check, BoolTest::eq));
2972 IfNode* iff = create_and_xform_if(control(), bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
2973 set_control(_gvn.transform(new IfTrueNode(iff)));
2974 return _gvn.transform(new IfFalseNode(iff));
2975 }
2976
2977 // Profile-driven exact type check:
2978 Node* GraphKit::type_check_receiver(Node* receiver, ciKlass* klass,
2979 float prob,
2980 Node* *casted_receiver) {
2981 assert(!klass->is_interface(), "no exact type check on interfaces");
2982
2983 const TypeKlassPtr* tklass = TypeKlassPtr::make(klass);
2984 Node* recv_klass = load_object_klass(receiver);
2985 Node* want_klass = makecon(tklass);
2986 Node* cmp = _gvn.transform(new CmpPNode(recv_klass, want_klass));
2987 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
2988 IfNode* iff = create_and_xform_if(control(), bol, prob, COUNT_UNKNOWN);
2989 set_control( _gvn.transform(new IfTrueNode (iff)));
2990 Node* fail = _gvn.transform(new IfFalseNode(iff));
2991
2992 if (!stopped()) {
2993 const TypeOopPtr* receiver_type = _gvn.type(receiver)->isa_oopptr();
2994 const TypeOopPtr* recvx_type = tklass->as_instance_type();
2995 assert(recvx_type->klass_is_exact(), "");
2996
2997 if (!receiver_type->higher_equal(recvx_type)) { // ignore redundant casts
2998 // Subsume downstream occurrences of receiver with a cast to
2999 // recv_xtype, since now we know what the type will be.
3000 Node* cast = new CheckCastPPNode(control(), receiver, recvx_type);
3001 (*casted_receiver) = _gvn.transform(cast);
3002 // (User must make the replace_in_map call.)
3003 }
3004 }
3005
3006 return fail;
3007 }
3008
3009 //------------------------------subtype_check_receiver-------------------------
3010 Node* GraphKit::subtype_check_receiver(Node* receiver, ciKlass* klass,
3011 Node** casted_receiver) {
3012 const TypeKlassPtr* tklass = TypeKlassPtr::make(klass);
3013 Node* want_klass = makecon(tklass);
3014
3015 Node* slow_ctl = gen_subtype_check(receiver, want_klass);
3016
3017 // Ignore interface type information until interface types are properly tracked.
3018 if (!stopped() && !klass->is_interface()) {
3019 const TypeOopPtr* receiver_type = _gvn.type(receiver)->isa_oopptr();
3020 const TypeOopPtr* recv_type = tklass->cast_to_exactness(false)->is_klassptr()->as_instance_type();
3021 if (!receiver_type->higher_equal(recv_type)) { // ignore redundant casts
3022 Node* cast = new CheckCastPPNode(control(), receiver, recv_type);
3023 (*casted_receiver) = _gvn.transform(cast);
3024 }
3025 }
3026
3027 return slow_ctl;
3028 }
3029
3030 //------------------------------seems_never_null-------------------------------
3031 // Use null_seen information if it is available from the profile.
3032 // If we see an unexpected null at a type check we record it and force a
3033 // recompile; the offending check will be recompiled to handle NULLs.
3034 // If we see several offending BCIs, then all checks in the
3035 // method will be recompiled.
3036 bool GraphKit::seems_never_null(Node* obj, ciProfileData* data, bool& speculating) {
3037 speculating = !_gvn.type(obj)->speculative_maybe_null();
3038 Deoptimization::DeoptReason reason = Deoptimization::reason_null_check(speculating);
3039 if (UncommonNullCast // Cutout for this technique
3040 && obj != null() // And not the -Xcomp stupid case?
3041 && !too_many_traps(reason)
3042 ) {
3043 if (speculating) {
3044 return true;
3045 }
3046 if (data == NULL)
3047 // Edge case: no mature data. Be optimistic here.
3048 return true;
3049 // If the profile has not seen a null, assume it won't happen.
3050 assert(java_bc() == Bytecodes::_checkcast ||
3051 java_bc() == Bytecodes::_instanceof ||
3052 java_bc() == Bytecodes::_aastore, "MDO must collect null_seen bit here");
3053 return !data->as_BitData()->null_seen();
3054 }
3055 speculating = false;
3056 return false;
3057 }
3058
3059 void GraphKit::guard_klass_being_initialized(Node* klass) {
3060 int init_state_off = in_bytes(InstanceKlass::init_state_offset());
3061 Node* adr = basic_plus_adr(top(), klass, init_state_off);
3062 Node* init_state = LoadNode::make(_gvn, NULL, immutable_memory(), adr,
3063 adr->bottom_type()->is_ptr(), TypeInt::BYTE,
3064 T_BYTE, MemNode::unordered);
3065 init_state = _gvn.transform(init_state);
3066
3067 Node* being_initialized_state = makecon(TypeInt::make(InstanceKlass::being_initialized));
3068
3069 Node* chk = _gvn.transform(new CmpINode(being_initialized_state, init_state));
3070 Node* tst = _gvn.transform(new BoolNode(chk, BoolTest::eq));
3071
3072 { BuildCutout unless(this, tst, PROB_MAX);
3112
3113 //------------------------maybe_cast_profiled_receiver-------------------------
3114 // If the profile has seen exactly one type, narrow to exactly that type.
3115 // Subsequent type checks will always fold up.
3116 Node* GraphKit::maybe_cast_profiled_receiver(Node* not_null_obj,
3117 ciKlass* require_klass,
3118 ciKlass* spec_klass,
3119 bool safe_for_replace) {
3120 if (!UseTypeProfile || !TypeProfileCasts) return NULL;
3121
3122 Deoptimization::DeoptReason reason = Deoptimization::reason_class_check(spec_klass != NULL);
3123
3124 // Make sure we haven't already deoptimized from this tactic.
3125 if (too_many_traps_or_recompiles(reason))
3126 return NULL;
3127
3128 // (No, this isn't a call, but it's enough like a virtual call
3129 // to use the same ciMethod accessor to get the profile info...)
3130 // If we have a speculative type use it instead of profiling (which
3131 // may not help us)
3132 ciKlass* exact_kls = spec_klass == NULL ? profile_has_unique_klass() : spec_klass;
3133 if (exact_kls != NULL) {// no cast failures here
3134 if (require_klass == NULL ||
3135 C->static_subtype_check(require_klass, exact_kls) == Compile::SSC_always_true) {
3136 // If we narrow the type to match what the type profile sees or
3137 // the speculative type, we can then remove the rest of the
3138 // cast.
3139 // This is a win, even if the exact_kls is very specific,
3140 // because downstream operations, such as method calls,
3141 // will often benefit from the sharper type.
3142 Node* exact_obj = not_null_obj; // will get updated in place...
3143 Node* slow_ctl = type_check_receiver(exact_obj, exact_kls, 1.0,
3144 &exact_obj);
3145 { PreserveJVMState pjvms(this);
3146 set_control(slow_ctl);
3147 uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
3148 }
3149 if (safe_for_replace) {
3150 replace_in_map(not_null_obj, exact_obj);
3151 }
3152 return exact_obj;
3217 // and the reflective instance-of call.
3218 Node* GraphKit::gen_instanceof(Node* obj, Node* superklass, bool safe_for_replace) {
3219 kill_dead_locals(); // Benefit all the uncommon traps
3220 assert( !stopped(), "dead parse path should be checked in callers" );
3221 assert(!TypePtr::NULL_PTR->higher_equal(_gvn.type(superklass)->is_klassptr()),
3222 "must check for not-null not-dead klass in callers");
3223
3224 // Make the merge point
3225 enum { _obj_path = 1, _fail_path, _null_path, PATH_LIMIT };
3226 RegionNode* region = new RegionNode(PATH_LIMIT);
3227 Node* phi = new PhiNode(region, TypeInt::BOOL);
3228 C->set_has_split_ifs(true); // Has chance for split-if optimization
3229
3230 ciProfileData* data = NULL;
3231 if (java_bc() == Bytecodes::_instanceof) { // Only for the bytecode
3232 data = method()->method_data()->bci_to_data(bci());
3233 }
3234 bool speculative_not_null = false;
3235 bool never_see_null = (ProfileDynamicTypes // aggressive use of profile
3236 && seems_never_null(obj, data, speculative_not_null));
3237
3238 // Null check; get casted pointer; set region slot 3
3239 Node* null_ctl = top();
3240 Node* not_null_obj = null_check_oop(obj, &null_ctl, never_see_null, safe_for_replace, speculative_not_null);
3241
3242 // If not_null_obj is dead, only null-path is taken
3243 if (stopped()) { // Doing instance-of on a NULL?
3244 set_control(null_ctl);
3245 return intcon(0);
3246 }
3247 region->init_req(_null_path, null_ctl);
3248 phi ->init_req(_null_path, intcon(0)); // Set null path value
3249 if (null_ctl == top()) {
3250 // Do this eagerly, so that pattern matches like is_diamond_phi
3251 // will work even during parsing.
3252 assert(_null_path == PATH_LIMIT-1, "delete last");
3253 region->del_req(_null_path);
3254 phi ->del_req(_null_path);
3255 }
3256
3257 // Do we know the type check always succeed?
3258 bool known_statically = false;
3259 if (_gvn.type(superklass)->singleton()) {
3260 ciKlass* superk = _gvn.type(superklass)->is_klassptr()->klass();
3261 ciKlass* subk = _gvn.type(obj)->is_oopptr()->klass();
3262 if (subk != NULL && subk->is_loaded()) {
3263 int static_res = C->static_subtype_check(superk, subk);
3264 known_statically = (static_res == Compile::SSC_always_true || static_res == Compile::SSC_always_false);
3265 }
3266 }
3267
3268 if (!known_statically) {
3269 const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr();
3270 // We may not have profiling here or it may not help us. If we
3271 // have a speculative type use it to perform an exact cast.
3272 ciKlass* spec_obj_type = obj_type->speculative_type();
3273 if (spec_obj_type != NULL || (ProfileDynamicTypes && data != NULL)) {
3274 Node* cast_obj = maybe_cast_profiled_receiver(not_null_obj, NULL, spec_obj_type, safe_for_replace);
3275 if (stopped()) { // Profile disagrees with this path.
3276 set_control(null_ctl); // Null is the only remaining possibility.
3277 return intcon(0);
3278 }
3279 if (cast_obj != NULL) {
3280 not_null_obj = cast_obj;
3281 }
3282 }
3283 }
3284
3285 // Generate the subtype check
3286 Node* not_subtype_ctrl = gen_subtype_check(not_null_obj, superklass);
3287
3288 // Plug in the success path to the general merge in slot 1.
3289 region->init_req(_obj_path, control());
3290 phi ->init_req(_obj_path, intcon(1));
3291
3292 // Plug in the failing path to the general merge in slot 2.
3293 region->init_req(_fail_path, not_subtype_ctrl);
3294 phi ->init_req(_fail_path, intcon(0));
3295
3296 // Return final merged results
3297 set_control( _gvn.transform(region) );
3298 record_for_igvn(region);
3299
3300 // If we know the type check always succeeds then we don't use the
3301 // profiling data at this bytecode. Don't lose it, feed it to the
3302 // type system as a speculative type.
3303 if (safe_for_replace) {
3304 Node* casted_obj = record_profiled_receiver_for_speculation(obj);
3305 replace_in_map(obj, casted_obj);
3306 }
3307
3308 return _gvn.transform(phi);
3309 }
3310
3311 //-------------------------------gen_checkcast---------------------------------
3312 // Generate a checkcast idiom. Used by both the checkcast bytecode and the
3313 // array store bytecode. Stack must be as-if BEFORE doing the bytecode so the
3314 // uncommon-trap paths work. Adjust stack after this call.
3315 // If failure_control is supplied and not null, it is filled in with
3316 // the control edge for the cast failure. Otherwise, an appropriate
3317 // uncommon trap or exception is thrown.
3318 Node* GraphKit::gen_checkcast(Node *obj, Node* superklass,
3319 Node* *failure_control) {
3320 kill_dead_locals(); // Benefit all the uncommon traps
3321 const TypeKlassPtr *tk = _gvn.type(superklass)->is_klassptr();
3322 const Type *toop = TypeOopPtr::make_from_klass(tk->klass());
3323
3324 // Fast cutout: Check the case that the cast is vacuously true.
3325 // This detects the common cases where the test will short-circuit
3326 // away completely. We do this before we perform the null check,
3327 // because if the test is going to turn into zero code, we don't
3328 // want a residual null check left around. (Causes a slowdown,
3329 // for example, in some objArray manipulations, such as a[i]=a[j].)
3330 if (tk->singleton()) {
3331 const TypeOopPtr* objtp = _gvn.type(obj)->isa_oopptr();
3332 if (objtp != NULL && objtp->klass() != NULL) {
3333 switch (C->static_subtype_check(tk->klass(), objtp->klass())) {
3334 case Compile::SSC_always_true:
3335 // If we know the type check always succeed then we don't use
3336 // the profiling data at this bytecode. Don't lose it, feed it
3337 // to the type system as a speculative type.
3338 return record_profiled_receiver_for_speculation(obj);
3339 case Compile::SSC_always_false:
3340 // It needs a null check because a null will *pass* the cast check.
3341 // A non-null value will always produce an exception.
3342 if (!objtp->maybe_null()) {
3343 builtin_throw(Deoptimization::Reason_class_check, makecon(TypeKlassPtr::make(objtp->klass())));
3344 return top();
3345 } else if (!too_many_traps_or_recompiles(Deoptimization::Reason_null_assert)) {
3346 return null_assert(obj);
3347 }
3348 break; // Fall through to full check
3349 }
3350 }
3351 }
3352
3353 ciProfileData* data = NULL;
3354 bool safe_for_replace = false;
3355 if (failure_control == NULL) { // use MDO in regular case only
3356 assert(java_bc() == Bytecodes::_aastore ||
3357 java_bc() == Bytecodes::_checkcast,
3358 "interpreter profiles type checks only for these BCs");
3359 data = method()->method_data()->bci_to_data(bci());
3360 safe_for_replace = true;
3361 }
3362
3363 // Make the merge point
3364 enum { _obj_path = 1, _null_path, PATH_LIMIT };
3365 RegionNode* region = new RegionNode(PATH_LIMIT);
3366 Node* phi = new PhiNode(region, toop);
3367 C->set_has_split_ifs(true); // Has chance for split-if optimization
3368
3369 // Use null-cast information if it is available
3370 bool speculative_not_null = false;
3371 bool never_see_null = ((failure_control == NULL) // regular case only
3372 && seems_never_null(obj, data, speculative_not_null));
3373
3374 // Null check; get casted pointer; set region slot 3
3375 Node* null_ctl = top();
3376 Node* not_null_obj = null_check_oop(obj, &null_ctl, never_see_null, safe_for_replace, speculative_not_null);
3377
3378 // If not_null_obj is dead, only null-path is taken
3379 if (stopped()) { // Doing instance-of on a NULL?
3380 set_control(null_ctl);
3381 return null();
3382 }
3383 region->init_req(_null_path, null_ctl);
3384 phi ->init_req(_null_path, null()); // Set null path value
3385 if (null_ctl == top()) {
3386 // Do this eagerly, so that pattern matches like is_diamond_phi
3387 // will work even during parsing.
3388 assert(_null_path == PATH_LIMIT-1, "delete last");
3389 region->del_req(_null_path);
3390 phi ->del_req(_null_path);
3391 }
3392
3393 Node* cast_obj = NULL;
3394 if (tk->klass_is_exact()) {
3395 // The following optimization tries to statically cast the speculative type of the object
3396 // (for example obtained during profiling) to the type of the superklass and then do a
3397 // dynamic check that the type of the object is what we expect. To work correctly
3398 // for checkcast and aastore the type of superklass should be exact.
3399 const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr();
3400 // We may not have profiling here or it may not help us. If we have
3401 // a speculative type use it to perform an exact cast.
3402 ciKlass* spec_obj_type = obj_type->speculative_type();
3403 if (spec_obj_type != NULL || data != NULL) {
3404 cast_obj = maybe_cast_profiled_receiver(not_null_obj, tk->klass(), spec_obj_type, safe_for_replace);
3405 if (cast_obj != NULL) {
3406 if (failure_control != NULL) // failure is now impossible
3407 (*failure_control) = top();
3408 // adjust the type of the phi to the exact klass:
3409 phi->raise_bottom_type(_gvn.type(cast_obj)->meet_speculative(TypePtr::NULL_PTR));
3410 }
3411 }
3412 }
3413
3414 if (cast_obj == NULL) {
3415 // Generate the subtype check
3416 Node* not_subtype_ctrl = gen_subtype_check(not_null_obj, superklass );
3417
3418 // Plug in success path into the merge
3419 cast_obj = _gvn.transform(new CheckCastPPNode(control(), not_null_obj, toop));
3420 // Failure path ends in uncommon trap (or may be dead - failure impossible)
3421 if (failure_control == NULL) {
3422 if (not_subtype_ctrl != top()) { // If failure is possible
3423 PreserveJVMState pjvms(this);
3424 set_control(not_subtype_ctrl);
3425 builtin_throw(Deoptimization::Reason_class_check, load_object_klass(not_null_obj));
3426 }
3427 } else {
3428 (*failure_control) = not_subtype_ctrl;
3429 }
3430 }
3431
3432 region->init_req(_obj_path, control());
3433 phi ->init_req(_obj_path, cast_obj);
3434
3435 // A merge of NULL or Casted-NotNull obj
3436 Node* res = _gvn.transform(phi);
3437
3438 // Note I do NOT always 'replace_in_map(obj,result)' here.
3439 // if( tk->klass()->can_be_primary_super() )
3440 // This means that if I successfully store an Object into an array-of-String
3441 // I 'forget' that the Object is really now known to be a String. I have to
3442 // do this because we don't have true union types for interfaces - if I store
3443 // a Baz into an array-of-Interface and then tell the optimizer it's an
3444 // Interface, I forget that it's also a Baz and cannot do Baz-like field
3445 // references to it. FIX THIS WHEN UNION TYPES APPEAR!
3446 // replace_in_map( obj, res );
3447
3448 // Return final merged results
3449 set_control( _gvn.transform(region) );
3450 record_for_igvn(region);
3451
3452 return record_profiled_receiver_for_speculation(res);
3453 }
3454
3455 //------------------------------next_monitor-----------------------------------
3456 // What number should be given to the next monitor?
3457 int GraphKit::next_monitor() {
3458 int current = jvms()->monitor_depth()* C->sync_stack_slots();
3459 int next = current + C->sync_stack_slots();
3460 // Keep the toplevel high water mark current:
3461 if (C->fixed_slots() < next) C->set_fixed_slots(next);
3462 return current;
3463 }
3464
3465 //------------------------------insert_mem_bar---------------------------------
3466 // Memory barrier to avoid floating things around
3467 // The membar serves as a pinch point between both control and all memory slices.
3468 Node* GraphKit::insert_mem_bar(int opcode, Node* precedent) {
3469 MemBarNode* mb = MemBarNode::make(C, opcode, Compile::AliasIdxBot, precedent);
3470 mb->init_req(TypeFunc::Control, control());
3471 mb->init_req(TypeFunc::Memory, reset_memory());
3472 Node* membar = _gvn.transform(mb);
3500 }
3501 Node* membar = _gvn.transform(mb);
3502 set_control(_gvn.transform(new ProjNode(membar, TypeFunc::Control)));
3503 if (alias_idx == Compile::AliasIdxBot) {
3504 merged_memory()->set_base_memory(_gvn.transform(new ProjNode(membar, TypeFunc::Memory)));
3505 } else {
3506 set_memory(_gvn.transform(new ProjNode(membar, TypeFunc::Memory)),alias_idx);
3507 }
3508 return membar;
3509 }
3510
3511 //------------------------------shared_lock------------------------------------
3512 // Emit locking code.
3513 FastLockNode* GraphKit::shared_lock(Node* obj) {
3514 // bci is either a monitorenter bc or InvocationEntryBci
3515 // %%% SynchronizationEntryBCI is redundant; use InvocationEntryBci in interfaces
3516 assert(SynchronizationEntryBCI == InvocationEntryBci, "");
3517
3518 if( !GenerateSynchronizationCode )
3519 return NULL; // Not locking things?
3520 if (stopped()) // Dead monitor?
3521 return NULL;
3522
3523 assert(dead_locals_are_killed(), "should kill locals before sync. point");
3524
3525 // Box the stack location
3526 Node* box = _gvn.transform(new BoxLockNode(next_monitor()));
3527 Node* mem = reset_memory();
3528
3529 FastLockNode * flock = _gvn.transform(new FastLockNode(0, obj, box) )->as_FastLock();
3530
3531 // Create the rtm counters for this fast lock if needed.
3532 flock->create_rtm_lock_counter(sync_jvms()); // sync_jvms used to get current bci
3533
3534 // Add monitor to debug info for the slow path. If we block inside the
3535 // slow path and de-opt, we need the monitor hanging around
3536 map()->push_monitor( flock );
3537
3538 const TypeFunc *tf = LockNode::lock_type();
3539 LockNode *lock = new LockNode(C, tf);
3568 }
3569 #endif
3570
3571 return flock;
3572 }
3573
3574
3575 //------------------------------shared_unlock----------------------------------
3576 // Emit unlocking code.
3577 void GraphKit::shared_unlock(Node* box, Node* obj) {
3578 // bci is either a monitorenter bc or InvocationEntryBci
3579 // %%% SynchronizationEntryBCI is redundant; use InvocationEntryBci in interfaces
3580 assert(SynchronizationEntryBCI == InvocationEntryBci, "");
3581
3582 if( !GenerateSynchronizationCode )
3583 return;
3584 if (stopped()) { // Dead monitor?
3585 map()->pop_monitor(); // Kill monitor from debug info
3586 return;
3587 }
3588
3589 // Memory barrier to avoid floating things down past the locked region
3590 insert_mem_bar(Op_MemBarReleaseLock);
3591
3592 const TypeFunc *tf = OptoRuntime::complete_monitor_exit_Type();
3593 UnlockNode *unlock = new UnlockNode(C, tf);
3594 #ifdef ASSERT
3595 unlock->set_dbg_jvms(sync_jvms());
3596 #endif
3597 uint raw_idx = Compile::AliasIdxRaw;
3598 unlock->init_req( TypeFunc::Control, control() );
3599 unlock->init_req( TypeFunc::Memory , memory(raw_idx) );
3600 unlock->init_req( TypeFunc::I_O , top() ) ; // does no i/o
3601 unlock->init_req( TypeFunc::FramePtr, frameptr() );
3602 unlock->init_req( TypeFunc::ReturnAdr, top() );
3603
3604 unlock->init_req(TypeFunc::Parms + 0, obj);
3605 unlock->init_req(TypeFunc::Parms + 1, box);
3606 unlock = _gvn.transform(unlock)->as_Unlock();
3607
3608 Node* mem = reset_memory();
3609
3610 // unlock has no side-effects, sets few values
3611 set_predefined_output_for_runtime_call(unlock, mem, TypeRawPtr::BOTTOM);
3612
3613 // Kill monitor from debug info
3614 map()->pop_monitor( );
3615 }
3616
3617 //-------------------------------get_layout_helper-----------------------------
3618 // If the given klass is a constant or known to be an array,
3619 // fetch the constant layout helper value into constant_value
3620 // and return (Node*)NULL. Otherwise, load the non-constant
3621 // layout helper value, and return the node which represents it.
3622 // This two-faced routine is useful because allocation sites
3623 // almost always feature constant types.
3624 Node* GraphKit::get_layout_helper(Node* klass_node, jint& constant_value) {
3625 const TypeKlassPtr* inst_klass = _gvn.type(klass_node)->isa_klassptr();
3626 if (!StressReflectiveCode && inst_klass != NULL) {
3627 ciKlass* klass = inst_klass->klass();
3628 bool xklass = inst_klass->klass_is_exact();
3629 if (xklass || klass->is_array_klass()) {
3630 jint lhelper = klass->layout_helper();
3631 if (lhelper != Klass::_lh_neutral_value) {
3632 constant_value = lhelper;
3633 return (Node*) NULL;
3634 }
3635 }
3636 }
3637 constant_value = Klass::_lh_neutral_value; // put in a known value
3638 Node* lhp = basic_plus_adr(klass_node, klass_node, in_bytes(Klass::layout_helper_offset()));
3639 return make_load(NULL, lhp, TypeInt::INT, T_INT, MemNode::unordered);
3640 }
3641
3642 // We just put in an allocate/initialize with a big raw-memory effect.
3643 // Hook selected additional alias categories on the initialization.
3644 static void hook_memory_on_init(GraphKit& kit, int alias_idx,
3645 MergeMemNode* init_in_merge,
3646 Node* init_out_raw) {
3647 DEBUG_ONLY(Node* init_in_raw = init_in_merge->base_memory());
3648 assert(init_in_merge->memory_at(alias_idx) == init_in_raw, "");
3649
3650 Node* prevmem = kit.memory(alias_idx);
3651 init_in_merge->set_memory_at(alias_idx, prevmem);
3652 kit.set_memory(init_out_raw, alias_idx);
3653 }
3654
3655 //---------------------------set_output_for_allocation-------------------------
3656 Node* GraphKit::set_output_for_allocation(AllocateNode* alloc,
3657 const TypeOopPtr* oop_type,
3658 bool deoptimize_on_exception) {
3659 int rawidx = Compile::AliasIdxRaw;
3660 alloc->set_req( TypeFunc::FramePtr, frameptr() );
3661 add_safepoint_edges(alloc);
3662 Node* allocx = _gvn.transform(alloc);
3663 set_control( _gvn.transform(new ProjNode(allocx, TypeFunc::Control) ) );
3664 // create memory projection for i_o
3665 set_memory ( _gvn.transform( new ProjNode(allocx, TypeFunc::Memory, true) ), rawidx );
3666 make_slow_call_ex(allocx, env()->Throwable_klass(), true, deoptimize_on_exception);
3667
3668 // create a memory projection as for the normal control path
3669 Node* malloc = _gvn.transform(new ProjNode(allocx, TypeFunc::Memory));
3670 set_memory(malloc, rawidx);
3671
3672 // a normal slow-call doesn't change i_o, but an allocation does
3673 // we create a separate i_o projection for the normal control path
3674 set_i_o(_gvn.transform( new ProjNode(allocx, TypeFunc::I_O, false) ) );
3675 Node* rawoop = _gvn.transform( new ProjNode(allocx, TypeFunc::Parms) );
3676
3677 // put in an initialization barrier
3678 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, rawidx,
3679 rawoop)->as_Initialize();
3680 assert(alloc->initialization() == init, "2-way macro link must work");
3681 assert(init ->allocation() == alloc, "2-way macro link must work");
3682 {
3683 // Extract memory strands which may participate in the new object's
3684 // initialization, and source them from the new InitializeNode.
3685 // This will allow us to observe initializations when they occur,
3686 // and link them properly (as a group) to the InitializeNode.
3687 assert(init->in(InitializeNode::Memory) == malloc, "");
3688 MergeMemNode* minit_in = MergeMemNode::make(malloc);
3689 init->set_req(InitializeNode::Memory, minit_in);
3690 record_for_igvn(minit_in); // fold it up later, if possible
3691 Node* minit_out = memory(rawidx);
3692 assert(minit_out->is_Proj() && minit_out->in(0) == init, "");
3693 // Add an edge in the MergeMem for the header fields so an access
3694 // to one of those has correct memory state
3695 set_memory(minit_out, C->get_alias_index(oop_type->add_offset(oopDesc::mark_offset_in_bytes())));
3696 set_memory(minit_out, C->get_alias_index(oop_type->add_offset(oopDesc::klass_offset_in_bytes())));
3697 if (oop_type->isa_aryptr()) {
3698 const TypePtr* telemref = oop_type->add_offset(Type::OffsetBot);
3699 int elemidx = C->get_alias_index(telemref);
3700 hook_memory_on_init(*this, elemidx, minit_in, minit_out);
3701 } else if (oop_type->isa_instptr()) {
3702 ciInstanceKlass* ik = oop_type->klass()->as_instance_klass();
3703 for (int i = 0, len = ik->nof_nonstatic_fields(); i < len; i++) {
3704 ciField* field = ik->nonstatic_field_at(i);
3705 if (field->offset() >= TrackedInitializationLimit * HeapWordSize)
3706 continue; // do not bother to track really large numbers of fields
3707 // Find (or create) the alias category for this field:
3708 int fieldidx = C->alias_type(field)->index();
3709 hook_memory_on_init(*this, fieldidx, minit_in, minit_out);
3710 }
3711 }
3712 }
3713
3714 // Cast raw oop to the real thing...
3715 Node* javaoop = new CheckCastPPNode(control(), rawoop, oop_type);
3716 javaoop = _gvn.transform(javaoop);
3717 C->set_recent_alloc(control(), javaoop);
3718 assert(just_allocated_object(control()) == javaoop, "just allocated");
3719
3720 #ifdef ASSERT
3721 { // Verify that the AllocateNode::Ideal_allocation recognizers work:
3732 assert(alloc->in(AllocateNode::ALength)->is_top(), "no length, please");
3733 }
3734 }
3735 #endif //ASSERT
3736
3737 return javaoop;
3738 }
3739
3740 //---------------------------new_instance--------------------------------------
3741 // This routine takes a klass_node which may be constant (for a static type)
3742 // or may be non-constant (for reflective code). It will work equally well
3743 // for either, and the graph will fold nicely if the optimizer later reduces
3744 // the type to a constant.
3745 // The optional arguments are for specialized use by intrinsics:
3746 // - If 'extra_slow_test' if not null is an extra condition for the slow-path.
3747 // - If 'return_size_val', report the the total object size to the caller.
3748 // - deoptimize_on_exception controls how Java exceptions are handled (rethrow vs deoptimize)
3749 Node* GraphKit::new_instance(Node* klass_node,
3750 Node* extra_slow_test,
3751 Node* *return_size_val,
3752 bool deoptimize_on_exception) {
3753 // Compute size in doublewords
3754 // The size is always an integral number of doublewords, represented
3755 // as a positive bytewise size stored in the klass's layout_helper.
3756 // The layout_helper also encodes (in a low bit) the need for a slow path.
3757 jint layout_con = Klass::_lh_neutral_value;
3758 Node* layout_val = get_layout_helper(klass_node, layout_con);
3759 int layout_is_con = (layout_val == NULL);
3760
3761 if (extra_slow_test == NULL) extra_slow_test = intcon(0);
3762 // Generate the initial go-slow test. It's either ALWAYS (return a
3763 // Node for 1) or NEVER (return a NULL) or perhaps (in the reflective
3764 // case) a computed value derived from the layout_helper.
3765 Node* initial_slow_test = NULL;
3766 if (layout_is_con) {
3767 assert(!StressReflectiveCode, "stress mode does not use these paths");
3768 bool must_go_slow = Klass::layout_helper_needs_slow_path(layout_con);
3769 initial_slow_test = must_go_slow ? intcon(1) : extra_slow_test;
3770 } else { // reflective case
3771 // This reflective path is used by Unsafe.allocateInstance.
3772 // (It may be stress-tested by specifying StressReflectiveCode.)
3773 // Basically, we want to get into the VM is there's an illegal argument.
3774 Node* bit = intcon(Klass::_lh_instance_slow_path_bit);
3775 initial_slow_test = _gvn.transform( new AndINode(layout_val, bit) );
3776 if (extra_slow_test != intcon(0)) {
3777 initial_slow_test = _gvn.transform( new OrINode(initial_slow_test, extra_slow_test) );
3778 }
3779 // (Macro-expander will further convert this to a Bool, if necessary.)
3790
3791 // Clear the low bits to extract layout_helper_size_in_bytes:
3792 assert((int)Klass::_lh_instance_slow_path_bit < BytesPerLong, "clear bit");
3793 Node* mask = MakeConX(~ (intptr_t)right_n_bits(LogBytesPerLong));
3794 size = _gvn.transform( new AndXNode(size, mask) );
3795 }
3796 if (return_size_val != NULL) {
3797 (*return_size_val) = size;
3798 }
3799
3800 // This is a precise notnull oop of the klass.
3801 // (Actually, it need not be precise if this is a reflective allocation.)
3802 // It's what we cast the result to.
3803 const TypeKlassPtr* tklass = _gvn.type(klass_node)->isa_klassptr();
3804 if (!tklass) tklass = TypeInstKlassPtr::OBJECT;
3805 const TypeOopPtr* oop_type = tklass->as_instance_type();
3806
3807 // Now generate allocation code
3808
3809 // The entire memory state is needed for slow path of the allocation
3810 // since GC and deoptimization can happened.
3811 Node *mem = reset_memory();
3812 set_all_memory(mem); // Create new memory state
3813
3814 AllocateNode* alloc = new AllocateNode(C, AllocateNode::alloc_type(Type::TOP),
3815 control(), mem, i_o(),
3816 size, klass_node,
3817 initial_slow_test);
3818
3819 return set_output_for_allocation(alloc, oop_type, deoptimize_on_exception);
3820 }
3821
3822 //-------------------------------new_array-------------------------------------
3823 // helper for both newarray and anewarray
3824 // The 'length' parameter is (obviously) the length of the array.
3825 // See comments on new_instance for the meaning of the other arguments.
3826 Node* GraphKit::new_array(Node* klass_node, // array klass (maybe variable)
3827 Node* length, // number of array elements
3828 int nargs, // number of arguments to push back for uncommon trap
3829 Node* *return_size_val,
3830 bool deoptimize_on_exception) {
3831 jint layout_con = Klass::_lh_neutral_value;
3832 Node* layout_val = get_layout_helper(klass_node, layout_con);
3833 int layout_is_con = (layout_val == NULL);
3834
3835 if (!layout_is_con && !StressReflectiveCode &&
3836 !too_many_traps(Deoptimization::Reason_class_check)) {
3837 // This is a reflective array creation site.
3838 // Optimistically assume that it is a subtype of Object[],
3839 // so that we can fold up all the address arithmetic.
3840 layout_con = Klass::array_layout_helper(T_OBJECT);
3841 Node* cmp_lh = _gvn.transform( new CmpINode(layout_val, intcon(layout_con)) );
3842 Node* bol_lh = _gvn.transform( new BoolNode(cmp_lh, BoolTest::eq) );
3843 { BuildCutout unless(this, bol_lh, PROB_MAX);
3844 inc_sp(nargs);
3845 uncommon_trap(Deoptimization::Reason_class_check,
3846 Deoptimization::Action_maybe_recompile);
3847 }
3848 layout_val = NULL;
3849 layout_is_con = true;
3850 }
3851
3852 // Generate the initial go-slow test. Make sure we do not overflow
3853 // if length is huge (near 2Gig) or negative! We do not need
3854 // exact double-words here, just a close approximation of needed
3855 // double-words. We can't add any offset or rounding bits, lest we
3856 // take a size -1 of bytes and make it positive. Use an unsigned
3857 // compare, so negative sizes look hugely positive.
3858 int fast_size_limit = FastAllocateSizeLimit;
3859 if (layout_is_con) {
3860 assert(!StressReflectiveCode, "stress mode does not use these paths");
3861 // Increase the size limit if we have exact knowledge of array type.
3862 int log2_esize = Klass::layout_helper_log2_element_size(layout_con);
3863 fast_size_limit <<= (LogBytesPerLong - log2_esize);
3864 }
3865
3866 Node* initial_slow_cmp = _gvn.transform( new CmpUNode( length, intcon( fast_size_limit ) ) );
3867 Node* initial_slow_test = _gvn.transform( new BoolNode( initial_slow_cmp, BoolTest::gt ) );
3868
3869 // --- Size Computation ---
3870 // array_size = round_to_heap(array_header + (length << elem_shift));
3871 // where round_to_heap(x) == align_to(x, MinObjAlignmentInBytes)
3872 // and align_to(x, y) == ((x + y-1) & ~(y-1))
3873 // The rounding mask is strength-reduced, if possible.
3874 int round_mask = MinObjAlignmentInBytes - 1;
3875 Node* header_size = NULL;
3876 int header_size_min = arrayOopDesc::base_offset_in_bytes(T_BYTE);
3877 // (T_BYTE has the weakest alignment and size restrictions...)
3878 if (layout_is_con) {
3879 int hsize = Klass::layout_helper_header_size(layout_con);
3880 int eshift = Klass::layout_helper_log2_element_size(layout_con);
3881 BasicType etype = Klass::layout_helper_element_type(layout_con);
3882 if ((round_mask & ~right_n_bits(eshift)) == 0)
3883 round_mask = 0; // strength-reduce it if it goes away completely
3884 assert((hsize & right_n_bits(eshift)) == 0, "hsize is pre-rounded");
3885 assert(header_size_min <= hsize, "generic minimum is smallest");
3886 header_size_min = hsize;
3887 header_size = intcon(hsize + round_mask);
3888 } else {
3889 Node* hss = intcon(Klass::_lh_header_size_shift);
3890 Node* hsm = intcon(Klass::_lh_header_size_mask);
3891 Node* hsize = _gvn.transform( new URShiftINode(layout_val, hss) );
3892 hsize = _gvn.transform( new AndINode(hsize, hsm) );
3893 Node* mask = intcon(round_mask);
3894 header_size = _gvn.transform( new AddINode(hsize, mask) );
3895 }
3896
3897 Node* elem_shift = NULL;
3898 if (layout_is_con) {
3899 int eshift = Klass::layout_helper_log2_element_size(layout_con);
3900 if (eshift != 0)
3901 elem_shift = intcon(eshift);
3902 } else {
3903 // There is no need to mask or shift this value.
3904 // The semantics of LShiftINode include an implicit mask to 0x1F.
3948 // places, one where the length is sharply limited, and the other
3949 // after a successful allocation.
3950 Node* abody = lengthx;
3951 if (elem_shift != NULL)
3952 abody = _gvn.transform( new LShiftXNode(lengthx, elem_shift) );
3953 Node* size = _gvn.transform( new AddXNode(headerx, abody) );
3954 if (round_mask != 0) {
3955 Node* mask = MakeConX(~round_mask);
3956 size = _gvn.transform( new AndXNode(size, mask) );
3957 }
3958 // else if round_mask == 0, the size computation is self-rounding
3959
3960 if (return_size_val != NULL) {
3961 // This is the size
3962 (*return_size_val) = size;
3963 }
3964
3965 // Now generate allocation code
3966
3967 // The entire memory state is needed for slow path of the allocation
3968 // since GC and deoptimization can happened.
3969 Node *mem = reset_memory();
3970 set_all_memory(mem); // Create new memory state
3971
3972 if (initial_slow_test->is_Bool()) {
3973 // Hide it behind a CMoveI, or else PhaseIdealLoop::split_up will get sick.
3974 initial_slow_test = initial_slow_test->as_Bool()->as_int_value(&_gvn);
3975 }
3976
3977 // Create the AllocateArrayNode and its result projections
3978 AllocateArrayNode* alloc
3979 = new AllocateArrayNode(C, AllocateArrayNode::alloc_type(TypeInt::INT),
3980 control(), mem, i_o(),
3981 size, klass_node,
3982 initial_slow_test,
3983 length);
3984
3985 // Cast to correct type. Note that the klass_node may be constant or not,
3986 // and in the latter case the actual array type will be inexact also.
3987 // (This happens via a non-constant argument to inline_native_newArray.)
3988 // In any case, the value of klass_node provides the desired array type.
3989 const TypeInt* length_type = _gvn.find_int_type(length);
3990 const TypeOopPtr* ary_type = _gvn.type(klass_node)->is_klassptr()->as_instance_type();
3991 if (ary_type->isa_aryptr() && length_type != NULL) {
3992 // Try to get a better type than POS for the size
3993 ary_type = ary_type->is_aryptr()->cast_to_size(length_type);
3994 }
3995
3996 Node* javaoop = set_output_for_allocation(alloc, ary_type, deoptimize_on_exception);
3997
3998 array_ideal_length(alloc, ary_type, true);
3999 return javaoop;
4000 }
4001
4002 // The following "Ideal_foo" functions are placed here because they recognize
4003 // the graph shapes created by the functions immediately above.
4004
4005 //---------------------------Ideal_allocation----------------------------------
4006 // Given an oop pointer or raw pointer, see if it feeds from an AllocateNode.
4007 AllocateNode* AllocateNode::Ideal_allocation(Node* ptr, PhaseTransform* phase) {
4008 if (ptr == NULL) { // reduce dumb test in callers
4009 return NULL;
4010 }
4119 set_all_memory(ideal.merged_memory());
4120 set_i_o(ideal.i_o());
4121 set_control(ideal.ctrl());
4122 }
4123
4124 void GraphKit::final_sync(IdealKit& ideal) {
4125 // Final sync IdealKit and graphKit.
4126 sync_kit(ideal);
4127 }
4128
4129 Node* GraphKit::load_String_length(Node* str, bool set_ctrl) {
4130 Node* len = load_array_length(load_String_value(str, set_ctrl));
4131 Node* coder = load_String_coder(str, set_ctrl);
4132 // Divide length by 2 if coder is UTF16
4133 return _gvn.transform(new RShiftINode(len, coder));
4134 }
4135
4136 Node* GraphKit::load_String_value(Node* str, bool set_ctrl) {
4137 int value_offset = java_lang_String::value_offset();
4138 const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4139 false, NULL, 0);
4140 const TypePtr* value_field_type = string_type->add_offset(value_offset);
4141 const TypeAryPtr* value_type = TypeAryPtr::make(TypePtr::NotNull,
4142 TypeAry::make(TypeInt::BYTE, TypeInt::POS),
4143 ciTypeArrayKlass::make(T_BYTE), true, 0);
4144 Node* p = basic_plus_adr(str, str, value_offset);
4145 Node* load = access_load_at(str, p, value_field_type, value_type, T_OBJECT,
4146 IN_HEAP | (set_ctrl ? C2_CONTROL_DEPENDENT_LOAD : 0) | MO_UNORDERED);
4147 return load;
4148 }
4149
4150 Node* GraphKit::load_String_coder(Node* str, bool set_ctrl) {
4151 if (!CompactStrings) {
4152 return intcon(java_lang_String::CODER_UTF16);
4153 }
4154 int coder_offset = java_lang_String::coder_offset();
4155 const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4156 false, NULL, 0);
4157 const TypePtr* coder_field_type = string_type->add_offset(coder_offset);
4158
4159 Node* p = basic_plus_adr(str, str, coder_offset);
4160 Node* load = access_load_at(str, p, coder_field_type, TypeInt::BYTE, T_BYTE,
4161 IN_HEAP | (set_ctrl ? C2_CONTROL_DEPENDENT_LOAD : 0) | MO_UNORDERED);
4162 return load;
4163 }
4164
4165 void GraphKit::store_String_value(Node* str, Node* value) {
4166 int value_offset = java_lang_String::value_offset();
4167 const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4168 false, NULL, 0);
4169 const TypePtr* value_field_type = string_type->add_offset(value_offset);
4170
4171 access_store_at(str, basic_plus_adr(str, value_offset), value_field_type,
4172 value, TypeAryPtr::BYTES, T_OBJECT, IN_HEAP | MO_UNORDERED);
4173 }
4174
4175 void GraphKit::store_String_coder(Node* str, Node* value) {
4176 int coder_offset = java_lang_String::coder_offset();
4177 const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4178 false, NULL, 0);
4179 const TypePtr* coder_field_type = string_type->add_offset(coder_offset);
4180
4181 access_store_at(str, basic_plus_adr(str, coder_offset), coder_field_type,
4182 value, TypeInt::BYTE, T_BYTE, IN_HEAP | MO_UNORDERED);
4183 }
4184
4185 // Capture src and dst memory state with a MergeMemNode
4186 Node* GraphKit::capture_memory(const TypePtr* src_type, const TypePtr* dst_type) {
4187 if (src_type == dst_type) {
4188 // Types are equal, we don't need a MergeMemNode
4189 return memory(src_type);
4190 }
4191 MergeMemNode* merge = MergeMemNode::make(map()->memory());
4192 record_for_igvn(merge); // fold it up later, if possible
4193 int src_idx = C->get_alias_index(src_type);
4194 int dst_idx = C->get_alias_index(dst_type);
4195 merge->set_memory_at(src_idx, memory(src_idx));
4196 merge->set_memory_at(dst_idx, memory(dst_idx));
4197 return merge;
4198 }
4271 i_char->init_req(2, AddI(i_char, intcon(2)));
4272
4273 set_control(IfFalse(iff));
4274 set_memory(st, TypeAryPtr::BYTES);
4275 }
4276
4277 Node* GraphKit::make_constant_from_field(ciField* field, Node* obj) {
4278 if (!field->is_constant()) {
4279 return NULL; // Field not marked as constant.
4280 }
4281 ciInstance* holder = NULL;
4282 if (!field->is_static()) {
4283 ciObject* const_oop = obj->bottom_type()->is_oopptr()->const_oop();
4284 if (const_oop != NULL && const_oop->is_instance()) {
4285 holder = const_oop->as_instance();
4286 }
4287 }
4288 const Type* con_type = Type::make_constant_from_field(field, holder, field->layout_type(),
4289 /*is_unsigned_load=*/false);
4290 if (con_type != NULL) {
4291 return makecon(con_type);
4292 }
4293 return NULL;
4294 }
|
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "ci/ciInlineKlass.hpp"
28 #include "ci/ciUtilities.hpp"
29 #include "classfile/javaClasses.hpp"
30 #include "ci/ciNativeEntryPoint.hpp"
31 #include "ci/ciObjArray.hpp"
32 #include "asm/register.hpp"
33 #include "compiler/compileLog.hpp"
34 #include "gc/shared/barrierSet.hpp"
35 #include "gc/shared/c2/barrierSetC2.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "memory/resourceArea.hpp"
38 #include "opto/addnode.hpp"
39 #include "opto/castnode.hpp"
40 #include "opto/convertnode.hpp"
41 #include "opto/graphKit.hpp"
42 #include "opto/idealKit.hpp"
43 #include "opto/inlinetypenode.hpp"
44 #include "opto/intrinsicnode.hpp"
45 #include "opto/locknode.hpp"
46 #include "opto/machnode.hpp"
47 #include "opto/narrowptrnode.hpp"
48 #include "opto/opaquenode.hpp"
49 #include "opto/parse.hpp"
50 #include "opto/rootnode.hpp"
51 #include "opto/runtime.hpp"
52 #include "opto/subtypenode.hpp"
53 #include "runtime/deoptimization.hpp"
54 #include "runtime/sharedRuntime.hpp"
55 #include "utilities/bitMap.inline.hpp"
56 #include "utilities/powerOfTwo.hpp"
57 #include "utilities/growableArray.hpp"
58
59 //----------------------------GraphKit-----------------------------------------
60 // Main utility constructor.
61 GraphKit::GraphKit(JVMState* jvms, PhaseGVN* gvn)
62 : Phase(Phase::Parser),
63 _env(C->env()),
64 _gvn((gvn != NULL) ? *gvn : *C->initial_gvn()),
65 _barrier_set(BarrierSet::barrier_set()->barrier_set_c2())
66 {
67 assert(gvn == NULL || !gvn->is_IterGVN() || gvn->is_IterGVN()->delay_transform(), "delay transform should be enabled");
68 _exceptions = jvms->map()->next_exception();
69 if (_exceptions != NULL) jvms->map()->set_next_exception(NULL);
70 set_jvms(jvms);
71 #ifdef ASSERT
72 if (_gvn.is_IterGVN() != NULL) {
73 assert(_gvn.is_IterGVN()->delay_transform(), "Transformation must be delayed if IterGVN is used");
74 // Save the initial size of _for_igvn worklist for verification (see ~GraphKit)
75 _worklist_size = _gvn.C->for_igvn()->size();
76 }
77 #endif
78 }
79
80 // Private constructor for parser.
81 GraphKit::GraphKit()
82 : Phase(Phase::Parser),
83 _env(C->env()),
84 _gvn(*C->initial_gvn()),
85 _barrier_set(BarrierSet::barrier_set()->barrier_set_c2())
86 {
87 _exceptions = NULL;
88 set_map(NULL);
89 debug_only(_sp = -99);
90 debug_only(set_bci(-99));
91 }
92
93
94
95 //---------------------------clean_stack---------------------------------------
96 // Clear away rubbish from the stack area of the JVM state.
97 // This destroys any arguments that may be waiting on the stack.
825 if (PrintMiscellaneous && (Verbose || WizardMode)) {
826 tty->print_cr("Zombie local %d: ", local);
827 jvms->dump();
828 }
829 return false;
830 }
831 }
832 }
833 return true;
834 }
835
836 #endif //ASSERT
837
838 // Helper function for enforcing certain bytecodes to reexecute if deoptimization happens.
839 static bool should_reexecute_implied_by_bytecode(JVMState *jvms, bool is_anewarray) {
840 ciMethod* cur_method = jvms->method();
841 int cur_bci = jvms->bci();
842 if (cur_method != NULL && cur_bci != InvocationEntryBci) {
843 Bytecodes::Code code = cur_method->java_code_at_bci(cur_bci);
844 return Interpreter::bytecode_should_reexecute(code) ||
845 (is_anewarray && (code == Bytecodes::_multianewarray));
846 // Reexecute _multianewarray bytecode which was replaced with
847 // sequence of [a]newarray. See Parse::do_multianewarray().
848 //
849 // Note: interpreter should not have it set since this optimization
850 // is limited by dimensions and guarded by flag so in some cases
851 // multianewarray() runtime calls will be generated and
852 // the bytecode should not be reexecutes (stack will not be reset).
853 } else {
854 return false;
855 }
856 }
857
858 // Helper function for adding JVMState and debug information to node
859 void GraphKit::add_safepoint_edges(SafePointNode* call, bool must_throw) {
860 // Add the safepoint edges to the call (or other safepoint).
861
862 // Make sure dead locals are set to top. This
863 // should help register allocation time and cut down on the size
864 // of the deoptimization information.
865 assert(dead_locals_are_killed(), "garbage in debug info before safepoint");
1085 ciSignature* declared_signature = NULL;
1086 ciMethod* ignored_callee = method()->get_method_at_bci(bci(), ignored_will_link, &declared_signature);
1087 assert(declared_signature != NULL, "cannot be null");
1088 inputs = declared_signature->arg_size_for_bc(code);
1089 int size = declared_signature->return_type()->size();
1090 depth = size - inputs;
1091 }
1092 break;
1093
1094 case Bytecodes::_multianewarray:
1095 {
1096 ciBytecodeStream iter(method());
1097 iter.reset_to_bci(bci());
1098 iter.next();
1099 inputs = iter.get_dimensions();
1100 assert(rsize == 1, "");
1101 depth = rsize - inputs;
1102 }
1103 break;
1104
1105 case Bytecodes::_withfield: {
1106 bool ignored_will_link;
1107 ciField* field = method()->get_field_at_bci(bci(), ignored_will_link);
1108 int size = field->type()->size();
1109 inputs = size+1;
1110 depth = rsize - inputs;
1111 break;
1112 }
1113
1114 case Bytecodes::_ireturn:
1115 case Bytecodes::_lreturn:
1116 case Bytecodes::_freturn:
1117 case Bytecodes::_dreturn:
1118 case Bytecodes::_areturn:
1119 assert(rsize == -depth, "");
1120 inputs = rsize;
1121 break;
1122
1123 case Bytecodes::_jsr:
1124 case Bytecodes::_jsr_w:
1125 inputs = 0;
1126 depth = 1; // S.B. depth=1, not zero
1127 break;
1128
1129 default:
1130 // bytecode produces a typed result
1131 inputs = rsize - depth;
1132 assert(inputs >= 0, "");
1133 break;
1176 Node* conv = _gvn.transform( new ConvI2LNode(offset));
1177 Node* mask = _gvn.transform(ConLNode::make((julong) max_juint));
1178 return _gvn.transform( new AndLNode(conv, mask) );
1179 }
1180
1181 Node* GraphKit::ConvL2I(Node* offset) {
1182 // short-circuit a common case
1183 jlong offset_con = find_long_con(offset, (jlong)Type::OffsetBot);
1184 if (offset_con != (jlong)Type::OffsetBot) {
1185 return intcon((int) offset_con);
1186 }
1187 return _gvn.transform( new ConvL2INode(offset));
1188 }
1189
1190 //-------------------------load_object_klass-----------------------------------
1191 Node* GraphKit::load_object_klass(Node* obj) {
1192 // Special-case a fresh allocation to avoid building nodes:
1193 Node* akls = AllocateNode::Ideal_klass(obj, &_gvn);
1194 if (akls != NULL) return akls;
1195 Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
1196 return _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), k_adr, TypeInstPtr::KLASS, TypeInstKlassPtr::OBJECT));
1197 }
1198
1199 //-------------------------load_array_length-----------------------------------
1200 Node* GraphKit::load_array_length(Node* array) {
1201 // Special-case a fresh allocation to avoid building nodes:
1202 AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(array, &_gvn);
1203 Node *alen;
1204 if (alloc == NULL) {
1205 Node *r_adr = basic_plus_adr(array, arrayOopDesc::length_offset_in_bytes());
1206 alen = _gvn.transform( new LoadRangeNode(0, immutable_memory(), r_adr, TypeInt::POS));
1207 } else {
1208 alen = array_ideal_length(alloc, _gvn.type(array)->is_oopptr(), false);
1209 }
1210 return alen;
1211 }
1212
1213 Node* GraphKit::array_ideal_length(AllocateArrayNode* alloc,
1214 const TypeOopPtr* oop_type,
1215 bool replace_length_in_map) {
1216 Node* length = alloc->Ideal_length();
1225 replace_in_map(length, ccast);
1226 }
1227 return ccast;
1228 }
1229 }
1230 return length;
1231 }
1232
1233 //------------------------------do_null_check----------------------------------
1234 // Helper function to do a NULL pointer check. Returned value is
1235 // the incoming address with NULL casted away. You are allowed to use the
1236 // not-null value only if you are control dependent on the test.
1237 #ifndef PRODUCT
1238 extern int explicit_null_checks_inserted,
1239 explicit_null_checks_elided;
1240 #endif
1241 Node* GraphKit::null_check_common(Node* value, BasicType type,
1242 // optional arguments for variations:
1243 bool assert_null,
1244 Node* *null_control,
1245 bool speculative,
1246 bool is_init_check) {
1247 assert(!assert_null || null_control == NULL, "not both at once");
1248 if (stopped()) return top();
1249 NOT_PRODUCT(explicit_null_checks_inserted++);
1250
1251 if (value->is_InlineTypePtr()) {
1252 // Null checking a scalarized but nullable inline type. Check the is_init
1253 // input instead of the oop input to avoid keeping buffer allocations alive.
1254 InlineTypePtrNode* vtptr = value->as_InlineTypePtr();
1255 while (vtptr->get_oop()->is_InlineTypePtr()) {
1256 vtptr = vtptr->get_oop()->as_InlineTypePtr();
1257 }
1258 null_check_common(vtptr->get_is_init(), T_INT, assert_null, null_control, speculative, true);
1259 if (stopped()) {
1260 return top();
1261 }
1262 bool do_replace_in_map = (null_control == NULL || (*null_control) == top());
1263 return cast_not_null(value, do_replace_in_map);
1264 }
1265
1266 // Construct NULL check
1267 Node *chk = NULL;
1268 switch(type) {
1269 case T_LONG : chk = new CmpLNode(value, _gvn.zerocon(T_LONG)); break;
1270 case T_INT : chk = new CmpINode(value, _gvn.intcon(0)); break;
1271 case T_INLINE_TYPE : // fall through
1272 case T_ARRAY : // fall through
1273 type = T_OBJECT; // simplify further tests
1274 case T_OBJECT : {
1275 const Type *t = _gvn.type( value );
1276
1277 const TypeOopPtr* tp = t->isa_oopptr();
1278 if (tp != NULL && tp->klass() != NULL && !tp->klass()->is_loaded()
1279 // Only for do_null_check, not any of its siblings:
1280 && !assert_null && null_control == NULL) {
1281 // Usually, any field access or invocation on an unloaded oop type
1282 // will simply fail to link, since the statically linked class is
1283 // likely also to be unloaded. However, in -Xcomp mode, sometimes
1284 // the static class is loaded but the sharper oop type is not.
1285 // Rather than checking for this obscure case in lots of places,
1286 // we simply observe that a null check on an unloaded class
1287 // will always be followed by a nonsense operation, so we
1288 // can just issue the uncommon trap here.
1289 // Our access to the unloaded class will only be correct
1290 // after it has been loaded and initialized, which requires
1291 // a trip through the interpreter.
1349 }
1350 Node *oldcontrol = control();
1351 set_control(cfg);
1352 Node *res = cast_not_null(value);
1353 set_control(oldcontrol);
1354 NOT_PRODUCT(explicit_null_checks_elided++);
1355 return res;
1356 }
1357 cfg = IfNode::up_one_dom(cfg, /*linear_only=*/ true);
1358 if (cfg == NULL) break; // Quit at region nodes
1359 depth++;
1360 }
1361 }
1362
1363 //-----------
1364 // Branch to failure if null
1365 float ok_prob = PROB_MAX; // a priori estimate: nulls never happen
1366 Deoptimization::DeoptReason reason;
1367 if (assert_null) {
1368 reason = Deoptimization::reason_null_assert(speculative);
1369 } else if (type == T_OBJECT || is_init_check) {
1370 reason = Deoptimization::reason_null_check(speculative);
1371 } else {
1372 reason = Deoptimization::Reason_div0_check;
1373 }
1374 // %%% Since Reason_unhandled is not recorded on a per-bytecode basis,
1375 // ciMethodData::has_trap_at will return a conservative -1 if any
1376 // must-be-null assertion has failed. This could cause performance
1377 // problems for a method after its first do_null_assert failure.
1378 // Consider using 'Reason_class_check' instead?
1379
1380 // To cause an implicit null check, we set the not-null probability
1381 // to the maximum (PROB_MAX). For an explicit check the probability
1382 // is set to a smaller value.
1383 if (null_control != NULL || too_many_traps(reason)) {
1384 // probability is less likely
1385 ok_prob = PROB_LIKELY_MAG(3);
1386 } else if (!assert_null &&
1387 (ImplicitNullCheckThreshold > 0) &&
1388 method() != NULL &&
1389 (method()->method_data()->trap_count(reason)
1423 }
1424
1425 if (assert_null) {
1426 // Cast obj to null on this path.
1427 replace_in_map(value, zerocon(type));
1428 return zerocon(type);
1429 }
1430
1431 // Cast obj to not-null on this path, if there is no null_control.
1432 // (If there is a null_control, a non-null value may come back to haunt us.)
1433 if (type == T_OBJECT) {
1434 Node* cast = cast_not_null(value, false);
1435 if (null_control == NULL || (*null_control) == top())
1436 replace_in_map(value, cast);
1437 value = cast;
1438 }
1439
1440 return value;
1441 }
1442
1443 //------------------------------cast_not_null----------------------------------
1444 // Cast obj to not-null on this path
1445 Node* GraphKit::cast_not_null(Node* obj, bool do_replace_in_map) {
1446 if (obj->is_InlineType()) {
1447 return obj;
1448 } else if (obj->is_InlineTypePtr()) {
1449 // Cast oop input instead
1450 Node* cast = cast_not_null(obj->as_InlineTypePtr()->get_oop(), do_replace_in_map);
1451 if (cast->is_top()) {
1452 // Always null
1453 return top();
1454 }
1455 // Create a new node with the casted oop input and is_init set
1456 InlineTypeBaseNode* vt = obj->clone()->as_InlineTypePtr();
1457 vt->set_oop(cast);
1458 vt->set_is_init(_gvn);
1459 vt = _gvn.transform(vt)->as_InlineTypePtr();
1460 if (do_replace_in_map) {
1461 replace_in_map(obj, vt);
1462 }
1463 return vt;
1464 }
1465 const Type *t = _gvn.type(obj);
1466 const Type *t_not_null = t->join_speculative(TypePtr::NOTNULL);
1467 // Object is already not-null?
1468 if( t == t_not_null ) return obj;
1469
1470 Node *cast = new CastPPNode(obj,t_not_null);
1471 cast->init_req(0, control());
1472 cast = _gvn.transform( cast );
1473
1474 // Scan for instances of 'obj' in the current JVM mapping.
1475 // These instances are known to be not-null after the test.
1476 if (do_replace_in_map)
1477 replace_in_map(obj, cast);
1478
1479 return cast; // Return casted value
1480 }
1481
1482 // Sometimes in intrinsics, we implicitly know an object is not null
1483 // (there's no actual null check) so we can cast it to not null. In
1484 // the course of optimizations, the input to the cast can become null.
1578 MemNode::MemOrd mo,
1579 LoadNode::ControlDependency control_dependency,
1580 bool require_atomic_access,
1581 bool unaligned,
1582 bool mismatched,
1583 bool unsafe,
1584 uint8_t barrier_data) {
1585 assert(adr_idx != Compile::AliasIdxTop, "use other make_load factory" );
1586 const TypePtr* adr_type = NULL; // debug-mode-only argument
1587 debug_only(adr_type = C->get_adr_type(adr_idx));
1588 Node* mem = memory(adr_idx);
1589 Node* ld;
1590 if (require_atomic_access && bt == T_LONG) {
1591 ld = LoadLNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data);
1592 } else if (require_atomic_access && bt == T_DOUBLE) {
1593 ld = LoadDNode::make_atomic(ctl, mem, adr, adr_type, t, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data);
1594 } else {
1595 ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt, mo, control_dependency, unaligned, mismatched, unsafe, barrier_data);
1596 }
1597 ld = _gvn.transform(ld);
1598
1599 if (((bt == T_OBJECT || bt == T_INLINE_TYPE) && C->do_escape_analysis()) || C->eliminate_boxing()) {
1600 // Improve graph before escape analysis and boxing elimination.
1601 record_for_igvn(ld);
1602 }
1603 return ld;
1604 }
1605
1606 Node* GraphKit::store_to_memory(Node* ctl, Node* adr, Node *val, BasicType bt,
1607 int adr_idx,
1608 MemNode::MemOrd mo,
1609 bool require_atomic_access,
1610 bool unaligned,
1611 bool mismatched,
1612 bool unsafe) {
1613 assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
1614 const TypePtr* adr_type = NULL;
1615 debug_only(adr_type = C->get_adr_type(adr_idx));
1616 Node *mem = memory(adr_idx);
1617 Node* st;
1618 if (require_atomic_access && bt == T_LONG) {
1619 st = StoreLNode::make_atomic(ctl, mem, adr, adr_type, val, mo);
1630 }
1631 if (unsafe) {
1632 st->as_Store()->set_unsafe_access();
1633 }
1634 st = _gvn.transform(st);
1635 set_memory(st, adr_idx);
1636 // Back-to-back stores can only remove intermediate store with DU info
1637 // so push on worklist for optimizer.
1638 if (mem->req() > MemNode::Address && adr == mem->in(MemNode::Address))
1639 record_for_igvn(st);
1640
1641 return st;
1642 }
1643
1644 Node* GraphKit::access_store_at(Node* obj,
1645 Node* adr,
1646 const TypePtr* adr_type,
1647 Node* val,
1648 const Type* val_type,
1649 BasicType bt,
1650 DecoratorSet decorators,
1651 bool safe_for_replace) {
1652 // Transformation of a value which could be NULL pointer (CastPP #NULL)
1653 // could be delayed during Parse (for example, in adjust_map_after_if()).
1654 // Execute transformation here to avoid barrier generation in such case.
1655 if (_gvn.type(val) == TypePtr::NULL_PTR) {
1656 val = _gvn.makecon(TypePtr::NULL_PTR);
1657 }
1658
1659 if (stopped()) {
1660 return top(); // Dead path ?
1661 }
1662
1663 assert(val != NULL, "not dead path");
1664 if (val->is_InlineType()) {
1665 // Store to non-flattened field. Buffer the inline type and make sure
1666 // the store is re-executed if the allocation triggers deoptimization.
1667 PreserveReexecuteState preexecs(this);
1668 jvms()->set_should_reexecute(true);
1669 val = val->as_InlineType()->buffer(this, safe_for_replace);
1670 }
1671
1672 C2AccessValuePtr addr(adr, adr_type);
1673 C2AccessValue value(val, val_type);
1674 C2ParseAccess access(this, decorators | C2_WRITE_ACCESS, bt, obj, addr);
1675 if (access.is_raw()) {
1676 return _barrier_set->BarrierSetC2::store_at(access, value);
1677 } else {
1678 return _barrier_set->store_at(access, value);
1679 }
1680 }
1681
1682 Node* GraphKit::access_load_at(Node* obj, // containing obj
1683 Node* adr, // actual adress to store val at
1684 const TypePtr* adr_type,
1685 const Type* val_type,
1686 BasicType bt,
1687 DecoratorSet decorators,
1688 Node* ctl) {
1689 if (stopped()) {
1690 return top(); // Dead path ?
1691 }
1692
1693 C2AccessValuePtr addr(adr, adr_type);
1694 C2ParseAccess access(this, decorators | C2_READ_ACCESS, bt, obj, addr, ctl);
1695 if (access.is_raw()) {
1696 return _barrier_set->BarrierSetC2::load_at(access, val_type);
1697 } else {
1698 return _barrier_set->load_at(access, val_type);
1699 }
1700 }
1701
1702 Node* GraphKit::access_load(Node* adr, // actual adress to load val at
1703 const Type* val_type,
1704 BasicType bt,
1705 DecoratorSet decorators) {
1706 if (stopped()) {
1707 return top(); // Dead path ?
1708 }
1709
1710 C2AccessValuePtr addr(adr, adr->bottom_type()->is_ptr());
1711 C2ParseAccess access(this, decorators | C2_READ_ACCESS, bt, NULL, addr);
1712 if (access.is_raw()) {
1713 return _barrier_set->BarrierSetC2::load_at(access, val_type);
1714 } else {
1780 const Type* value_type,
1781 BasicType bt,
1782 DecoratorSet decorators) {
1783 C2AccessValuePtr addr(adr, adr_type);
1784 C2AtomicParseAccess access(this, decorators | C2_READ_ACCESS | C2_WRITE_ACCESS, bt, obj, addr, alias_idx);
1785 if (access.is_raw()) {
1786 return _barrier_set->BarrierSetC2::atomic_add_at(access, new_val, value_type);
1787 } else {
1788 return _barrier_set->atomic_add_at(access, new_val, value_type);
1789 }
1790 }
1791
1792 void GraphKit::access_clone(Node* src, Node* dst, Node* size, bool is_array) {
1793 return _barrier_set->clone(this, src, dst, size, is_array);
1794 }
1795
1796 //-------------------------array_element_address-------------------------
1797 Node* GraphKit::array_element_address(Node* ary, Node* idx, BasicType elembt,
1798 const TypeInt* sizetype, Node* ctrl) {
1799 uint shift = exact_log2(type2aelembytes(elembt));
1800 ciKlass* arytype_klass = _gvn.type(ary)->is_aryptr()->klass();
1801 if (arytype_klass != NULL && arytype_klass->is_flat_array_klass()) {
1802 ciFlatArrayKlass* vak = arytype_klass->as_flat_array_klass();
1803 shift = vak->log2_element_size();
1804 }
1805 uint header = arrayOopDesc::base_offset_in_bytes(elembt);
1806
1807 // short-circuit a common case (saves lots of confusing waste motion)
1808 jint idx_con = find_int_con(idx, -1);
1809 if (idx_con >= 0) {
1810 intptr_t offset = header + ((intptr_t)idx_con << shift);
1811 return basic_plus_adr(ary, offset);
1812 }
1813
1814 // must be correct type for alignment purposes
1815 Node* base = basic_plus_adr(ary, header);
1816 idx = Compile::conv_I2X_index(&_gvn, idx, sizetype, ctrl);
1817 Node* scale = _gvn.transform( new LShiftXNode(idx, intcon(shift)) );
1818 return basic_plus_adr(ary, base, scale);
1819 }
1820
1821 //-------------------------load_array_element-------------------------
1822 Node* GraphKit::load_array_element(Node* ary, Node* idx, const TypeAryPtr* arytype, bool set_ctrl) {
1823 const Type* elemtype = arytype->elem();
1824 BasicType elembt = elemtype->array_element_basic_type();
1825 assert(elembt != T_INLINE_TYPE, "inline types are not supported by this method");
1826 Node* adr = array_element_address(ary, idx, elembt, arytype->size());
1827 if (elembt == T_NARROWOOP) {
1828 elembt = T_OBJECT; // To satisfy switch in LoadNode::make()
1829 }
1830 Node* ld = access_load_at(ary, adr, arytype, elemtype, elembt,
1831 IN_HEAP | IS_ARRAY | (set_ctrl ? C2_CONTROL_DEPENDENT_LOAD : 0));
1832 return ld;
1833 }
1834
1835 //-------------------------set_arguments_for_java_call-------------------------
1836 // Arguments (pre-popped from the stack) are taken from the JVMS.
1837 void GraphKit::set_arguments_for_java_call(CallJavaNode* call, bool is_late_inline) {
1838 PreserveReexecuteState preexecs(this);
1839 if (EnableValhalla) {
1840 // Make sure the call is "re-executed", if buffering of inline type arguments triggers deoptimization.
1841 // At this point, the call hasn't been executed yet, so we will only ever execute the call once.
1842 jvms()->set_should_reexecute(true);
1843 int arg_size = method()->get_declared_signature_at_bci(bci())->arg_size_for_bc(java_bc());
1844 inc_sp(arg_size);
1845 }
1846 // Add the call arguments
1847 const TypeTuple* domain = call->tf()->domain_sig();
1848 uint nargs = domain->cnt();
1849 for (uint i = TypeFunc::Parms, idx = TypeFunc::Parms; i < nargs; i++) {
1850 Node* arg = argument(i-TypeFunc::Parms);
1851 const Type* t = domain->field_at(i);
1852 if (call->method()->has_scalarized_args() && t->is_inlinetypeptr() && !t->maybe_null() && t->inline_klass()->can_be_passed_as_fields()) {
1853 // We don't pass inline type arguments by reference but instead pass each field of the inline type
1854 InlineTypeBaseNode* vt = arg->as_InlineTypeBase();
1855 vt->pass_fields(this, call, idx);
1856 // If an inline type argument is passed as fields, attach the Method* to the call site
1857 // to be able to access the extended signature later via attached_method_before_pc().
1858 // For example, see CompiledMethod::preserve_callee_argument_oops().
1859 call->set_override_symbolic_info(true);
1860 continue;
1861 } else if (arg->is_InlineType()) {
1862 // Pass inline type argument via oop to callee
1863 arg = arg->as_InlineType()->buffer(this);
1864 if (!is_late_inline) {
1865 arg = arg->as_InlineTypePtr()->get_oop();
1866 }
1867 }
1868 call->init_req(idx++, arg);
1869 }
1870 }
1871
1872 //---------------------------set_edges_for_java_call---------------------------
1873 // Connect a newly created call into the current JVMS.
1874 // A return value node (if any) is returned from set_edges_for_java_call.
1875 void GraphKit::set_edges_for_java_call(CallJavaNode* call, bool must_throw, bool separate_io_proj) {
1876
1877 // Add the predefined inputs:
1878 call->init_req( TypeFunc::Control, control() );
1879 call->init_req( TypeFunc::I_O , i_o() );
1880 call->init_req( TypeFunc::Memory , reset_memory() );
1881 call->init_req( TypeFunc::FramePtr, frameptr() );
1882 call->init_req( TypeFunc::ReturnAdr, top() );
1883
1884 add_safepoint_edges(call, must_throw);
1885
1886 Node* xcall = _gvn.transform(call);
1887
1888 if (xcall == top()) {
1889 set_control(top());
1890 return;
1891 }
1892 assert(xcall == call, "call identity is stable");
1893
1894 // Re-use the current map to produce the result.
1895
1896 set_control(_gvn.transform(new ProjNode(call, TypeFunc::Control)));
1897 set_i_o( _gvn.transform(new ProjNode(call, TypeFunc::I_O , separate_io_proj)));
1898 set_all_memory_call(xcall, separate_io_proj);
1899
1900 //return xcall; // no need, caller already has it
1901 }
1902
1903 Node* GraphKit::set_results_for_java_call(CallJavaNode* call, bool separate_io_proj, bool deoptimize) {
1904 if (stopped()) return top(); // maybe the call folded up?
1905
1906 // Note: Since any out-of-line call can produce an exception,
1907 // we always insert an I_O projection from the call into the result.
1908
1909 make_slow_call_ex(call, env()->Throwable_klass(), separate_io_proj, deoptimize);
1910
1911 if (separate_io_proj) {
1912 // The caller requested separate projections be used by the fall
1913 // through and exceptional paths, so replace the projections for
1914 // the fall through path.
1915 set_i_o(_gvn.transform( new ProjNode(call, TypeFunc::I_O) ));
1916 set_all_memory(_gvn.transform( new ProjNode(call, TypeFunc::Memory) ));
1917 }
1918
1919 // Capture the return value, if any.
1920 Node* ret;
1921 if (call->method() == NULL || call->method()->return_type()->basic_type() == T_VOID) {
1922 ret = top();
1923 } else if (call->method()->return_type()->is_inlinetype()) {
1924 const Type* ret_type = call->tf()->range_sig()->field_at(TypeFunc::Parms);
1925 if (call->tf()->returns_inline_type_as_fields()) {
1926 // Return of multiple values (inline type fields): we create a
1927 // InlineType node, each field is a projection from the call.
1928 ciInlineKlass* vk = call->method()->return_type()->as_inline_klass();
1929 uint base_input = TypeFunc::Parms;
1930 ret = InlineTypeNode::make_from_multi(this, call, ret_type->inline_klass(), base_input, false);
1931 } else {
1932 ret = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
1933 ret = _gvn.transform(InlineTypeNode::make_from_oop(this, ret, ret_type->inline_klass(), !ret_type->maybe_null()));
1934 }
1935 } else {
1936 ret = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
1937 }
1938
1939 return ret;
1940 }
1941
1942 //--------------------set_predefined_input_for_runtime_call--------------------
1943 // Reading and setting the memory state is way conservative here.
1944 // The real problem is that I am not doing real Type analysis on memory,
1945 // so I cannot distinguish card mark stores from other stores. Across a GC
1946 // point the Store Barrier and the card mark memory has to agree. I cannot
1947 // have a card mark store and its barrier split across the GC point from
1948 // either above or below. Here I get that to happen by reading ALL of memory.
1949 // A better answer would be to separate out card marks from other memory.
1950 // For now, return the input memory state, so that it can be reused
1951 // after the call, if this call has restricted memory effects.
1952 Node* GraphKit::set_predefined_input_for_runtime_call(SafePointNode* call, Node* narrow_mem) {
1953 // Set fixed predefined input arguments
1954 Node* memory = reset_memory();
1955 Node* m = narrow_mem == NULL ? memory : narrow_mem;
1956 call->init_req( TypeFunc::Control, control() );
1957 call->init_req( TypeFunc::I_O, top() ); // does no i/o
1958 call->init_req( TypeFunc::Memory, m ); // may gc ptrs
2009 if (use->is_MergeMem()) {
2010 wl.push(use);
2011 }
2012 }
2013 }
2014
2015 // Replace the call with the current state of the kit.
2016 void GraphKit::replace_call(CallNode* call, Node* result, bool do_replaced_nodes) {
2017 JVMState* ejvms = NULL;
2018 if (has_exceptions()) {
2019 ejvms = transfer_exceptions_into_jvms();
2020 }
2021
2022 ReplacedNodes replaced_nodes = map()->replaced_nodes();
2023 ReplacedNodes replaced_nodes_exception;
2024 Node* ex_ctl = top();
2025
2026 SafePointNode* final_state = stop();
2027
2028 // Find all the needed outputs of this call
2029 CallProjections* callprojs = call->extract_projections(true);
2030
2031 Unique_Node_List wl;
2032 Node* init_mem = call->in(TypeFunc::Memory);
2033 Node* final_mem = final_state->in(TypeFunc::Memory);
2034 Node* final_ctl = final_state->in(TypeFunc::Control);
2035 Node* final_io = final_state->in(TypeFunc::I_O);
2036
2037 // Replace all the old call edges with the edges from the inlining result
2038 if (callprojs->fallthrough_catchproj != NULL) {
2039 C->gvn_replace_by(callprojs->fallthrough_catchproj, final_ctl);
2040 }
2041 if (callprojs->fallthrough_memproj != NULL) {
2042 if (final_mem->is_MergeMem()) {
2043 // Parser's exits MergeMem was not transformed but may be optimized
2044 final_mem = _gvn.transform(final_mem);
2045 }
2046 C->gvn_replace_by(callprojs->fallthrough_memproj, final_mem);
2047 add_mergemem_users_to_worklist(wl, final_mem);
2048 }
2049 if (callprojs->fallthrough_ioproj != NULL) {
2050 C->gvn_replace_by(callprojs->fallthrough_ioproj, final_io);
2051 }
2052
2053 // Replace the result with the new result if it exists and is used
2054 if (callprojs->resproj[0] != NULL && result != NULL) {
2055 // If the inlined code is dead, the result projections for an inline type returned as
2056 // fields have not been replaced. They will go away once the call is replaced by TOP below.
2057 assert(callprojs->nb_resproj == 1 || (call->tf()->returns_inline_type_as_fields() && stopped()),
2058 "unexpected number of results");
2059 C->gvn_replace_by(callprojs->resproj[0], result);
2060 }
2061
2062 if (ejvms == NULL) {
2063 // No exception edges to simply kill off those paths
2064 if (callprojs->catchall_catchproj != NULL) {
2065 C->gvn_replace_by(callprojs->catchall_catchproj, C->top());
2066 }
2067 if (callprojs->catchall_memproj != NULL) {
2068 C->gvn_replace_by(callprojs->catchall_memproj, C->top());
2069 }
2070 if (callprojs->catchall_ioproj != NULL) {
2071 C->gvn_replace_by(callprojs->catchall_ioproj, C->top());
2072 }
2073 // Replace the old exception object with top
2074 if (callprojs->exobj != NULL) {
2075 C->gvn_replace_by(callprojs->exobj, C->top());
2076 }
2077 } else {
2078 GraphKit ekit(ejvms);
2079
2080 // Load my combined exception state into the kit, with all phis transformed:
2081 SafePointNode* ex_map = ekit.combine_and_pop_all_exception_states();
2082 replaced_nodes_exception = ex_map->replaced_nodes();
2083
2084 Node* ex_oop = ekit.use_exception_state(ex_map);
2085
2086 if (callprojs->catchall_catchproj != NULL) {
2087 C->gvn_replace_by(callprojs->catchall_catchproj, ekit.control());
2088 ex_ctl = ekit.control();
2089 }
2090 if (callprojs->catchall_memproj != NULL) {
2091 Node* ex_mem = ekit.reset_memory();
2092 C->gvn_replace_by(callprojs->catchall_memproj, ex_mem);
2093 add_mergemem_users_to_worklist(wl, ex_mem);
2094 }
2095 if (callprojs->catchall_ioproj != NULL) {
2096 C->gvn_replace_by(callprojs->catchall_ioproj, ekit.i_o());
2097 }
2098
2099 // Replace the old exception object with the newly created one
2100 if (callprojs->exobj != NULL) {
2101 C->gvn_replace_by(callprojs->exobj, ex_oop);
2102 }
2103 }
2104
2105 // Disconnect the call from the graph
2106 call->disconnect_inputs(C);
2107 C->gvn_replace_by(call, C->top());
2108
2109 // Clean up any MergeMems that feed other MergeMems since the
2110 // optimizer doesn't like that.
2111 while (wl.size() > 0) {
2112 _gvn.transform(wl.pop());
2113 }
2114
2115 if (callprojs->fallthrough_catchproj != NULL && !final_ctl->is_top() && do_replaced_nodes) {
2116 replaced_nodes.apply(C, final_ctl);
2117 }
2118 if (!ex_ctl->is_top() && do_replaced_nodes) {
2119 replaced_nodes_exception.apply(C, ex_ctl);
2120 }
2121 }
2122
2123
2124 //------------------------------increment_counter------------------------------
2125 // for statistics: increment a VM counter by 1
2126
2127 void GraphKit::increment_counter(address counter_addr) {
2128 Node* adr1 = makecon(TypeRawPtr::make(counter_addr));
2129 increment_counter(adr1);
2130 }
2131
2132 void GraphKit::increment_counter(Node* counter_addr) {
2133 int adr_type = Compile::AliasIdxRaw;
2134 Node* ctrl = control();
2135 Node* cnt = make_load(ctrl, counter_addr, TypeLong::LONG, T_LONG, adr_type, MemNode::unordered);
2293 *
2294 * @param n node that the type applies to
2295 * @param exact_kls type from profiling
2296 * @param maybe_null did profiling see null?
2297 *
2298 * @return node with improved type
2299 */
2300 Node* GraphKit::record_profile_for_speculation(Node* n, ciKlass* exact_kls, ProfilePtrKind ptr_kind) {
2301 const Type* current_type = _gvn.type(n);
2302 assert(UseTypeSpeculation, "type speculation must be on");
2303
2304 const TypePtr* speculative = current_type->speculative();
2305
2306 // Should the klass from the profile be recorded in the speculative type?
2307 if (current_type->would_improve_type(exact_kls, jvms()->depth())) {
2308 const TypeKlassPtr* tklass = TypeKlassPtr::make(exact_kls);
2309 const TypeOopPtr* xtype = tklass->as_instance_type();
2310 assert(xtype->klass_is_exact(), "Should be exact");
2311 // Any reason to believe n is not null (from this profiling or a previous one)?
2312 assert(ptr_kind != ProfileAlwaysNull, "impossible here");
2313 const TypePtr* ptr = (ptr_kind != ProfileNeverNull && current_type->speculative_maybe_null()) ? TypePtr::BOTTOM : TypePtr::NOTNULL;
2314 // record the new speculative type's depth
2315 speculative = xtype->cast_to_ptr_type(ptr->ptr())->is_ptr();
2316 speculative = speculative->with_inline_depth(jvms()->depth());
2317 } else if (current_type->would_improve_ptr(ptr_kind)) {
2318 // Profiling report that null was never seen so we can change the
2319 // speculative type to non null ptr.
2320 if (ptr_kind == ProfileAlwaysNull) {
2321 speculative = TypePtr::NULL_PTR;
2322 } else {
2323 assert(ptr_kind == ProfileNeverNull, "nothing else is an improvement");
2324 const TypePtr* ptr = TypePtr::NOTNULL;
2325 if (speculative != NULL) {
2326 speculative = speculative->cast_to_ptr_type(ptr->ptr())->is_ptr();
2327 } else {
2328 speculative = ptr;
2329 }
2330 }
2331 }
2332
2333 if (speculative != current_type->speculative()) {
2334 // Build a type with a speculative type (what we think we know
2335 // about the type but will need a guard when we use it)
2336 const TypeOopPtr* spec_type = TypeOopPtr::make(TypePtr::BotPTR, Type::Offset::bottom, TypeOopPtr::InstanceBot, speculative);
2337 // We're changing the type, we need a new CheckCast node to carry
2338 // the new type. The new type depends on the control: what
2339 // profiling tells us is only valid from here as far as we can
2340 // tell.
2341 Node* cast = new CheckCastPPNode(control(), n, current_type->remove_speculative()->join_speculative(spec_type));
2342 cast = _gvn.transform(cast);
2343 replace_in_map(n, cast);
2344 n = cast;
2345 }
2346
2347 return n;
2348 }
2349
2350 /**
2351 * Record profiling data from receiver profiling at an invoke with the
2352 * type system so that it can propagate it (speculation)
2353 *
2354 * @param n receiver node
2355 *
2356 * @return node with improved type
2357 */
2358 Node* GraphKit::record_profiled_receiver_for_speculation(Node* n) {
2359 if (!UseTypeSpeculation) {
2360 return n;
2361 }
2362 ciKlass* exact_kls = profile_has_unique_klass();
2363 ProfilePtrKind ptr_kind = ProfileMaybeNull;
2364 if ((java_bc() == Bytecodes::_checkcast ||
2365 java_bc() == Bytecodes::_instanceof ||
2366 java_bc() == Bytecodes::_aastore) &&
2367 method()->method_data()->is_mature()) {
2368 ciProfileData* data = method()->method_data()->bci_to_data(bci());
2369 if (data != NULL) {
2370 if (java_bc() == Bytecodes::_aastore) {
2371 ciKlass* array_type = NULL;
2372 ciKlass* element_type = NULL;
2373 ProfilePtrKind element_ptr = ProfileMaybeNull;
2374 bool flat_array = true;
2375 bool null_free_array = true;
2376 method()->array_access_profiled_type(bci(), array_type, element_type, element_ptr, flat_array, null_free_array);
2377 exact_kls = element_type;
2378 ptr_kind = element_ptr;
2379 } else {
2380 if (!data->as_BitData()->null_seen()) {
2381 ptr_kind = ProfileNeverNull;
2382 } else {
2383 assert(data->is_ReceiverTypeData(), "bad profile data type");
2384 ciReceiverTypeData* call = (ciReceiverTypeData*)data->as_ReceiverTypeData();
2385 uint i = 0;
2386 for (; i < call->row_limit(); i++) {
2387 ciKlass* receiver = call->receiver(i);
2388 if (receiver != NULL) {
2389 break;
2390 }
2391 }
2392 ptr_kind = (i == call->row_limit()) ? ProfileAlwaysNull : ProfileMaybeNull;
2393 }
2394 }
2395 }
2396 }
2397 return record_profile_for_speculation(n, exact_kls, ptr_kind);
2398 }
2399
2400 /**
2401 * Record profiling data from argument profiling at an invoke with the
2402 * type system so that it can propagate it (speculation)
2403 *
2404 * @param dest_method target method for the call
2405 * @param bc what invoke bytecode is this?
2406 */
2407 void GraphKit::record_profiled_arguments_for_speculation(ciMethod* dest_method, Bytecodes::Code bc) {
2408 if (!UseTypeSpeculation) {
2409 return;
2410 }
2411 const TypeFunc* tf = TypeFunc::make(dest_method);
2412 int nargs = tf->domain_sig()->cnt() - TypeFunc::Parms;
2413 int skip = Bytecodes::has_receiver(bc) ? 1 : 0;
2414 for (int j = skip, i = 0; j < nargs && i < TypeProfileArgsLimit; j++) {
2415 const Type *targ = tf->domain_sig()->field_at(j + TypeFunc::Parms);
2416 if (is_reference_type(targ->basic_type())) {
2417 ProfilePtrKind ptr_kind = ProfileMaybeNull;
2418 ciKlass* better_type = NULL;
2419 if (method()->argument_profiled_type(bci(), i, better_type, ptr_kind)) {
2420 record_profile_for_speculation(argument(j), better_type, ptr_kind);
2421 }
2422 i++;
2423 }
2424 }
2425 }
2426
2427 /**
2428 * Record profiling data from parameter profiling at an invoke with
2429 * the type system so that it can propagate it (speculation)
2430 */
2431 void GraphKit::record_profiled_parameters_for_speculation() {
2432 if (!UseTypeSpeculation) {
2433 return;
2434 }
2435 for (int i = 0, j = 0; i < method()->arg_size() ; i++) {
2449 * the type system so that it can propagate it (speculation)
2450 */
2451 void GraphKit::record_profiled_return_for_speculation() {
2452 if (!UseTypeSpeculation) {
2453 return;
2454 }
2455 ProfilePtrKind ptr_kind = ProfileMaybeNull;
2456 ciKlass* better_type = NULL;
2457 if (method()->return_profiled_type(bci(), better_type, ptr_kind)) {
2458 // If profiling reports a single type for the return value,
2459 // feed it to the type system so it can propagate it as a
2460 // speculative type
2461 record_profile_for_speculation(stack(sp()-1), better_type, ptr_kind);
2462 }
2463 }
2464
2465 void GraphKit::round_double_arguments(ciMethod* dest_method) {
2466 if (Matcher::strict_fp_requires_explicit_rounding) {
2467 // (Note: TypeFunc::make has a cache that makes this fast.)
2468 const TypeFunc* tf = TypeFunc::make(dest_method);
2469 int nargs = tf->domain_sig()->cnt() - TypeFunc::Parms;
2470 for (int j = 0; j < nargs; j++) {
2471 const Type *targ = tf->domain_sig()->field_at(j + TypeFunc::Parms);
2472 if (targ->basic_type() == T_DOUBLE) {
2473 // If any parameters are doubles, they must be rounded before
2474 // the call, dstore_rounding does gvn.transform
2475 Node *arg = argument(j);
2476 arg = dstore_rounding(arg);
2477 set_argument(j, arg);
2478 }
2479 }
2480 }
2481 }
2482
2483 // rounding for strict float precision conformance
2484 Node* GraphKit::precision_rounding(Node* n) {
2485 if (Matcher::strict_fp_requires_explicit_rounding) {
2486 #ifdef IA32
2487 if (UseSSE == 0) {
2488 return _gvn.transform(new RoundFloatNode(0, n));
2489 }
2490 #else
2491 Unimplemented();
2615 Node* parm0, Node* parm1,
2616 Node* parm2, Node* parm3,
2617 Node* parm4, Node* parm5,
2618 Node* parm6, Node* parm7,
2619 Node* parm8) {
2620 assert(call_addr != NULL, "must not call NULL targets");
2621
2622 // Slow-path call
2623 bool is_leaf = !(flags & RC_NO_LEAF);
2624 bool has_io = (!is_leaf && !(flags & RC_NO_IO));
2625 if (call_name == NULL) {
2626 assert(!is_leaf, "must supply name for leaf");
2627 call_name = OptoRuntime::stub_name(call_addr);
2628 }
2629 CallNode* call;
2630 if (!is_leaf) {
2631 call = new CallStaticJavaNode(call_type, call_addr, call_name, adr_type);
2632 } else if (flags & RC_NO_FP) {
2633 call = new CallLeafNoFPNode(call_type, call_addr, call_name, adr_type);
2634 } else if (flags & RC_VECTOR){
2635 uint num_bits = call_type->range_sig()->field_at(TypeFunc::Parms)->is_vect()->length_in_bytes() * BitsPerByte;
2636 call = new CallLeafVectorNode(call_type, call_addr, call_name, adr_type, num_bits);
2637 } else {
2638 call = new CallLeafNode(call_type, call_addr, call_name, adr_type);
2639 }
2640
2641 // The following is similar to set_edges_for_java_call,
2642 // except that the memory effects of the call are restricted to AliasIdxRaw.
2643
2644 // Slow path call has no side-effects, uses few values
2645 bool wide_in = !(flags & RC_NARROW_MEM);
2646 bool wide_out = (C->get_alias_index(adr_type) == Compile::AliasIdxBot);
2647
2648 Node* prev_mem = NULL;
2649 if (wide_in) {
2650 prev_mem = set_predefined_input_for_runtime_call(call);
2651 } else {
2652 assert(!wide_out, "narrow in => narrow out");
2653 Node* narrow_mem = memory(adr_type);
2654 prev_mem = set_predefined_input_for_runtime_call(call, narrow_mem);
2655 }
2715
2716 //-----------------------------make_native_call-------------------------------
2717 Node* GraphKit::make_native_call(address call_addr, const TypeFunc* call_type, uint nargs, ciNativeEntryPoint* nep) {
2718 // Select just the actual call args to pass on
2719 // [MethodHandle fallback, long addr, HALF addr, ... args , NativeEntryPoint nep]
2720 // | |
2721 // V V
2722 // [ ... args ]
2723 uint n_filtered_args = nargs - 4; // -fallback, -addr (2), -nep;
2724 ResourceMark rm;
2725 Node** argument_nodes = NEW_RESOURCE_ARRAY(Node*, n_filtered_args);
2726 const Type** arg_types = TypeTuple::fields(n_filtered_args);
2727 GrowableArray<VMReg> arg_regs(C->comp_arena(), n_filtered_args, n_filtered_args, VMRegImpl::Bad());
2728
2729 VMReg* argRegs = nep->argMoves();
2730 {
2731 for (uint vm_arg_pos = 0, java_arg_read_pos = 0;
2732 vm_arg_pos < n_filtered_args; vm_arg_pos++) {
2733 uint vm_unfiltered_arg_pos = vm_arg_pos + 3; // +3 to skip fallback handle argument and addr (2 since long)
2734 Node* node = argument(vm_unfiltered_arg_pos);
2735 const Type* type = call_type->domain_sig()->field_at(TypeFunc::Parms + vm_unfiltered_arg_pos);
2736 VMReg reg = type == Type::HALF
2737 ? VMRegImpl::Bad()
2738 : argRegs[java_arg_read_pos++];
2739
2740 argument_nodes[vm_arg_pos] = node;
2741 arg_types[TypeFunc::Parms + vm_arg_pos] = type;
2742 arg_regs.at_put(vm_arg_pos, reg);
2743 }
2744 }
2745
2746 uint n_returns = call_type->range_sig()->cnt() - TypeFunc::Parms;
2747 GrowableArray<VMReg> ret_regs(C->comp_arena(), n_returns, n_returns, VMRegImpl::Bad());
2748 const Type** ret_types = TypeTuple::fields(n_returns);
2749
2750 VMReg* retRegs = nep->returnMoves();
2751 {
2752 for (uint vm_ret_pos = 0, java_ret_read_pos = 0;
2753 vm_ret_pos < n_returns; vm_ret_pos++) { // 0 or 1
2754 const Type* type = call_type->range_sig()->field_at(TypeFunc::Parms + vm_ret_pos);
2755 VMReg reg = type == Type::HALF
2756 ? VMRegImpl::Bad()
2757 : retRegs[java_ret_read_pos++];
2758
2759 ret_regs.at_put(vm_ret_pos, reg);
2760 ret_types[TypeFunc::Parms + vm_ret_pos] = type;
2761 }
2762 }
2763
2764 const TypeFunc* new_call_type = TypeFunc::make(
2765 TypeTuple::make(TypeFunc::Parms + n_filtered_args, arg_types),
2766 TypeTuple::make(TypeFunc::Parms + n_returns, ret_types)
2767 );
2768
2769 if (nep->need_transition()) {
2770 RuntimeStub* invoker = SharedRuntime::make_native_invoker(call_addr,
2771 nep->shadow_space(),
2772 arg_regs, ret_regs);
2773 if (invoker == NULL) {
2774 C->record_failure("native invoker not implemented on this platform");
3061
3062 // Now do a linear scan of the secondary super-klass array. Again, no real
3063 // performance impact (too rare) but it's gotta be done.
3064 // Since the code is rarely used, there is no penalty for moving it
3065 // out of line, and it can only improve I-cache density.
3066 // The decision to inline or out-of-line this final check is platform
3067 // dependent, and is found in the AD file definition of PartialSubtypeCheck.
3068 Node* psc = gvn.transform(
3069 new PartialSubtypeCheckNode(*ctrl, subklass, superklass));
3070
3071 IfNode *iff4 = gen_subtype_check_compare(*ctrl, psc, gvn.zerocon(T_OBJECT), BoolTest::ne, PROB_FAIR, gvn, T_ADDRESS);
3072 r_not_subtype->init_req(2, gvn.transform(new IfTrueNode (iff4)));
3073 r_ok_subtype ->init_req(3, gvn.transform(new IfFalseNode(iff4)));
3074
3075 // Return false path; set default control to true path.
3076 *ctrl = gvn.transform(r_ok_subtype);
3077 return gvn.transform(r_not_subtype);
3078 }
3079
3080 Node* GraphKit::gen_subtype_check(Node* obj_or_subklass, Node* superklass) {
3081 const Type* sub_t = _gvn.type(obj_or_subklass);
3082 if (sub_t->isa_inlinetype()) {
3083 obj_or_subklass = makecon(TypeKlassPtr::make(sub_t->inline_klass()));
3084 }
3085 bool expand_subtype_check = C->post_loop_opts_phase() || // macro node expansion is over
3086 ExpandSubTypeCheckAtParseTime; // forced expansion
3087 if (expand_subtype_check) {
3088 MergeMemNode* mem = merged_memory();
3089 Node* ctrl = control();
3090 Node* subklass = obj_or_subklass;
3091 if (!sub_t->isa_klassptr() && !sub_t->isa_inlinetype()) {
3092 subklass = load_object_klass(obj_or_subklass);
3093 }
3094 Node* n = Phase::gen_subtype_check(subklass, superklass, &ctrl, mem, _gvn);
3095 set_control(ctrl);
3096 return n;
3097 }
3098
3099 Node* check = _gvn.transform(new SubTypeCheckNode(C, obj_or_subklass, superklass));
3100 Node* bol = _gvn.transform(new BoolNode(check, BoolTest::eq));
3101 IfNode* iff = create_and_xform_if(control(), bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
3102 set_control(_gvn.transform(new IfTrueNode(iff)));
3103 return _gvn.transform(new IfFalseNode(iff));
3104 }
3105
3106 // Profile-driven exact type check:
3107 Node* GraphKit::type_check_receiver(Node* receiver, ciKlass* klass,
3108 float prob, Node* *casted_receiver) {
3109 assert(!klass->is_interface(), "no exact type check on interfaces");
3110 Node* fail = top();
3111 const Type* rec_t = _gvn.type(receiver);
3112 if (rec_t->isa_inlinetype()) {
3113 if (klass->equals(rec_t->inline_klass())) {
3114 (*casted_receiver) = receiver; // Always passes
3115 } else {
3116 (*casted_receiver) = top(); // Always fails
3117 fail = control();
3118 set_control(top());
3119 }
3120 return fail;
3121 }
3122 const TypeKlassPtr* tklass = TypeKlassPtr::make(klass);
3123 Node* recv_klass = load_object_klass(receiver);
3124 fail = type_check(recv_klass, tklass, prob);
3125
3126 if (!stopped()) {
3127 const TypeOopPtr* receiver_type = _gvn.type(receiver)->isa_oopptr();
3128 const TypeOopPtr* recv_xtype = tklass->as_instance_type();
3129 assert(recv_xtype->klass_is_exact(), "");
3130
3131 if (!receiver_type->higher_equal(recv_xtype)) { // ignore redundant casts
3132 // Subsume downstream occurrences of receiver with a cast to
3133 // recv_xtype, since now we know what the type will be.
3134 Node* cast = new CheckCastPPNode(control(), receiver, recv_xtype);
3135 Node* res = _gvn.transform(cast);
3136 if (recv_xtype->is_inlinetypeptr()) {
3137 assert(!gvn().type(res)->maybe_null(), "receiver should never be null");
3138 res = InlineTypeNode::make_from_oop(this, res, recv_xtype->inline_klass())->as_InlineTypeBase()->as_ptr(&gvn());
3139 }
3140 (*casted_receiver) = res;
3141 // (User must make the replace_in_map call.)
3142 }
3143 }
3144
3145 return fail;
3146 }
3147
3148 Node* GraphKit::type_check(Node* recv_klass, const TypeKlassPtr* tklass,
3149 float prob) {
3150 Node* want_klass = makecon(tklass);
3151 Node* cmp = _gvn.transform(new CmpPNode(recv_klass, want_klass));
3152 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
3153 IfNode* iff = create_and_xform_if(control(), bol, prob, COUNT_UNKNOWN);
3154 set_control(_gvn.transform(new IfTrueNode (iff)));
3155 Node* fail = _gvn.transform(new IfFalseNode(iff));
3156 return fail;
3157 }
3158
3159 //------------------------------subtype_check_receiver-------------------------
3160 Node* GraphKit::subtype_check_receiver(Node* receiver, ciKlass* klass,
3161 Node** casted_receiver) {
3162 const TypeKlassPtr* tklass = TypeKlassPtr::make(klass);
3163 Node* want_klass = makecon(tklass);
3164
3165 Node* slow_ctl = gen_subtype_check(receiver, want_klass);
3166
3167 // Ignore interface type information until interface types are properly tracked.
3168 if (!stopped() && !klass->is_interface()) {
3169 const TypeOopPtr* receiver_type = _gvn.type(receiver)->isa_oopptr();
3170 const TypeOopPtr* recv_type = tklass->cast_to_exactness(false)->is_klassptr()->as_instance_type();
3171 if (receiver_type != NULL && !receiver_type->higher_equal(recv_type)) { // ignore redundant casts
3172 Node* cast = new CheckCastPPNode(control(), receiver, recv_type);
3173 (*casted_receiver) = _gvn.transform(cast);
3174 }
3175 }
3176
3177 return slow_ctl;
3178 }
3179
3180 //------------------------------seems_never_null-------------------------------
3181 // Use null_seen information if it is available from the profile.
3182 // If we see an unexpected null at a type check we record it and force a
3183 // recompile; the offending check will be recompiled to handle NULLs.
3184 // If we see several offending BCIs, then all checks in the
3185 // method will be recompiled.
3186 bool GraphKit::seems_never_null(Node* obj, ciProfileData* data, bool& speculating) {
3187 speculating = !_gvn.type(obj)->speculative_maybe_null();
3188 Deoptimization::DeoptReason reason = Deoptimization::reason_null_check(speculating);
3189 if (UncommonNullCast // Cutout for this technique
3190 && obj != null() // And not the -Xcomp stupid case?
3191 && !too_many_traps(reason)
3192 ) {
3193 if (speculating) {
3194 return true;
3195 }
3196 if (data == NULL)
3197 // Edge case: no mature data. Be optimistic here.
3198 return true;
3199 // If the profile has not seen a null, assume it won't happen.
3200 assert(java_bc() == Bytecodes::_checkcast ||
3201 java_bc() == Bytecodes::_instanceof ||
3202 java_bc() == Bytecodes::_aastore, "MDO must collect null_seen bit here");
3203 if (java_bc() == Bytecodes::_aastore) {
3204 return ((ciArrayLoadStoreData*)data->as_ArrayLoadStoreData())->element()->ptr_kind() == ProfileNeverNull;
3205 }
3206 return !data->as_BitData()->null_seen();
3207 }
3208 speculating = false;
3209 return false;
3210 }
3211
3212 void GraphKit::guard_klass_being_initialized(Node* klass) {
3213 int init_state_off = in_bytes(InstanceKlass::init_state_offset());
3214 Node* adr = basic_plus_adr(top(), klass, init_state_off);
3215 Node* init_state = LoadNode::make(_gvn, NULL, immutable_memory(), adr,
3216 adr->bottom_type()->is_ptr(), TypeInt::BYTE,
3217 T_BYTE, MemNode::unordered);
3218 init_state = _gvn.transform(init_state);
3219
3220 Node* being_initialized_state = makecon(TypeInt::make(InstanceKlass::being_initialized));
3221
3222 Node* chk = _gvn.transform(new CmpINode(being_initialized_state, init_state));
3223 Node* tst = _gvn.transform(new BoolNode(chk, BoolTest::eq));
3224
3225 { BuildCutout unless(this, tst, PROB_MAX);
3265
3266 //------------------------maybe_cast_profiled_receiver-------------------------
3267 // If the profile has seen exactly one type, narrow to exactly that type.
3268 // Subsequent type checks will always fold up.
3269 Node* GraphKit::maybe_cast_profiled_receiver(Node* not_null_obj,
3270 ciKlass* require_klass,
3271 ciKlass* spec_klass,
3272 bool safe_for_replace) {
3273 if (!UseTypeProfile || !TypeProfileCasts) return NULL;
3274
3275 Deoptimization::DeoptReason reason = Deoptimization::reason_class_check(spec_klass != NULL);
3276
3277 // Make sure we haven't already deoptimized from this tactic.
3278 if (too_many_traps_or_recompiles(reason))
3279 return NULL;
3280
3281 // (No, this isn't a call, but it's enough like a virtual call
3282 // to use the same ciMethod accessor to get the profile info...)
3283 // If we have a speculative type use it instead of profiling (which
3284 // may not help us)
3285 ciKlass* exact_kls = spec_klass;
3286 if (exact_kls == NULL) {
3287 if (java_bc() == Bytecodes::_aastore) {
3288 ciKlass* array_type = NULL;
3289 ciKlass* element_type = NULL;
3290 ProfilePtrKind element_ptr = ProfileMaybeNull;
3291 bool flat_array = true;
3292 bool null_free_array = true;
3293 method()->array_access_profiled_type(bci(), array_type, element_type, element_ptr, flat_array, null_free_array);
3294 exact_kls = element_type;
3295 } else {
3296 exact_kls = profile_has_unique_klass();
3297 }
3298 }
3299 if (exact_kls != NULL) {// no cast failures here
3300 if (require_klass == NULL ||
3301 C->static_subtype_check(require_klass, exact_kls) == Compile::SSC_always_true) {
3302 // If we narrow the type to match what the type profile sees or
3303 // the speculative type, we can then remove the rest of the
3304 // cast.
3305 // This is a win, even if the exact_kls is very specific,
3306 // because downstream operations, such as method calls,
3307 // will often benefit from the sharper type.
3308 Node* exact_obj = not_null_obj; // will get updated in place...
3309 Node* slow_ctl = type_check_receiver(exact_obj, exact_kls, 1.0,
3310 &exact_obj);
3311 { PreserveJVMState pjvms(this);
3312 set_control(slow_ctl);
3313 uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
3314 }
3315 if (safe_for_replace) {
3316 replace_in_map(not_null_obj, exact_obj);
3317 }
3318 return exact_obj;
3383 // and the reflective instance-of call.
3384 Node* GraphKit::gen_instanceof(Node* obj, Node* superklass, bool safe_for_replace) {
3385 kill_dead_locals(); // Benefit all the uncommon traps
3386 assert( !stopped(), "dead parse path should be checked in callers" );
3387 assert(!TypePtr::NULL_PTR->higher_equal(_gvn.type(superklass)->is_klassptr()),
3388 "must check for not-null not-dead klass in callers");
3389
3390 // Make the merge point
3391 enum { _obj_path = 1, _fail_path, _null_path, PATH_LIMIT };
3392 RegionNode* region = new RegionNode(PATH_LIMIT);
3393 Node* phi = new PhiNode(region, TypeInt::BOOL);
3394 C->set_has_split_ifs(true); // Has chance for split-if optimization
3395
3396 ciProfileData* data = NULL;
3397 if (java_bc() == Bytecodes::_instanceof) { // Only for the bytecode
3398 data = method()->method_data()->bci_to_data(bci());
3399 }
3400 bool speculative_not_null = false;
3401 bool never_see_null = (ProfileDynamicTypes // aggressive use of profile
3402 && seems_never_null(obj, data, speculative_not_null));
3403 bool is_value = obj->is_InlineType();
3404
3405 // Null check; get casted pointer; set region slot 3
3406 Node* null_ctl = top();
3407 Node* not_null_obj = is_value ? obj : null_check_oop(obj, &null_ctl, never_see_null, safe_for_replace, speculative_not_null);
3408
3409 // If not_null_obj is dead, only null-path is taken
3410 if (stopped()) { // Doing instance-of on a NULL?
3411 set_control(null_ctl);
3412 return intcon(0);
3413 }
3414 region->init_req(_null_path, null_ctl);
3415 phi ->init_req(_null_path, intcon(0)); // Set null path value
3416 if (null_ctl == top()) {
3417 // Do this eagerly, so that pattern matches like is_diamond_phi
3418 // will work even during parsing.
3419 assert(_null_path == PATH_LIMIT-1, "delete last");
3420 region->del_req(_null_path);
3421 phi ->del_req(_null_path);
3422 }
3423
3424 // Do we know the type check always succeed?
3425 if (!is_value) {
3426 bool known_statically = false;
3427 if (_gvn.type(superklass)->singleton()) {
3428 ciKlass* superk = _gvn.type(superklass)->is_klassptr()->klass();
3429 ciKlass* subk = _gvn.type(obj)->is_oopptr()->klass();
3430 if (subk != NULL && subk->is_loaded()) {
3431 int static_res = C->static_subtype_check(superk, subk);
3432 known_statically = (static_res == Compile::SSC_always_true || static_res == Compile::SSC_always_false);
3433 }
3434 }
3435
3436 if (!known_statically) {
3437 const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr();
3438 // We may not have profiling here or it may not help us. If we
3439 // have a speculative type use it to perform an exact cast.
3440 ciKlass* spec_obj_type = obj_type->speculative_type();
3441 if (spec_obj_type != NULL || (ProfileDynamicTypes && data != NULL)) {
3442 Node* cast_obj = maybe_cast_profiled_receiver(not_null_obj, NULL, spec_obj_type, safe_for_replace);
3443 if (stopped()) { // Profile disagrees with this path.
3444 set_control(null_ctl); // Null is the only remaining possibility.
3445 return intcon(0);
3446 }
3447 if (cast_obj != NULL &&
3448 // A value that's sometimes null is not something we can optimize well
3449 !(cast_obj->is_InlineType() && null_ctl != top())) {
3450 not_null_obj = cast_obj;
3451 is_value = not_null_obj->is_InlineType();
3452 }
3453 }
3454 }
3455 }
3456
3457 // Generate the subtype check
3458 Node* not_subtype_ctrl = gen_subtype_check(not_null_obj, superklass);
3459
3460 // Plug in the success path to the general merge in slot 1.
3461 region->init_req(_obj_path, control());
3462 phi ->init_req(_obj_path, intcon(1));
3463
3464 // Plug in the failing path to the general merge in slot 2.
3465 region->init_req(_fail_path, not_subtype_ctrl);
3466 phi ->init_req(_fail_path, intcon(0));
3467
3468 // Return final merged results
3469 set_control( _gvn.transform(region) );
3470 record_for_igvn(region);
3471
3472 // If we know the type check always succeeds then we don't use the
3473 // profiling data at this bytecode. Don't lose it, feed it to the
3474 // type system as a speculative type.
3475 if (safe_for_replace && !is_value) {
3476 Node* casted_obj = record_profiled_receiver_for_speculation(obj);
3477 replace_in_map(obj, casted_obj);
3478 }
3479
3480 return _gvn.transform(phi);
3481 }
3482
3483 //-------------------------------gen_checkcast---------------------------------
3484 // Generate a checkcast idiom. Used by both the checkcast bytecode and the
3485 // array store bytecode. Stack must be as-if BEFORE doing the bytecode so the
3486 // uncommon-trap paths work. Adjust stack after this call.
3487 // If failure_control is supplied and not null, it is filled in with
3488 // the control edge for the cast failure. Otherwise, an appropriate
3489 // uncommon trap or exception is thrown.
3490 Node* GraphKit::gen_checkcast(Node *obj, Node* superklass, Node* *failure_control, bool null_free) {
3491 kill_dead_locals(); // Benefit all the uncommon traps
3492 const TypeKlassPtr* tk = _gvn.type(superklass)->is_klassptr();
3493 const TypeOopPtr* toop = TypeOopPtr::make_from_klass(tk->klass());
3494 bool safe_for_replace = (failure_control == NULL);
3495 bool from_inline = obj->is_InlineType();
3496 assert(!null_free || toop->is_inlinetypeptr(), "must be an inline type pointer");
3497
3498 // Fast cutout: Check the case that the cast is vacuously true.
3499 // This detects the common cases where the test will short-circuit
3500 // away completely. We do this before we perform the null check,
3501 // because if the test is going to turn into zero code, we don't
3502 // want a residual null check left around. (Causes a slowdown,
3503 // for example, in some objArray manipulations, such as a[i]=a[j].)
3504 if (tk->singleton()) {
3505 ciKlass* klass = NULL;
3506 if (obj->is_InlineTypeBase()) {
3507 klass = _gvn.type(obj)->inline_klass();
3508 } else {
3509 const TypeOopPtr* objtp = _gvn.type(obj)->isa_oopptr();
3510 if (objtp != NULL) {
3511 klass = objtp->klass();
3512 }
3513 }
3514 if (klass != NULL) {
3515 switch (C->static_subtype_check(tk->klass(), klass)) {
3516 case Compile::SSC_always_true:
3517 // If we know the type check always succeed then we don't use
3518 // the profiling data at this bytecode. Don't lose it, feed it
3519 // to the type system as a speculative type.
3520 if (!from_inline) {
3521 obj = record_profiled_receiver_for_speculation(obj);
3522 if (null_free) {
3523 assert(safe_for_replace, "must be");
3524 obj = null_check(obj);
3525 }
3526 assert(stopped() || !toop->is_inlinetypeptr() ||
3527 obj->is_InlineTypeBase(), "should have been scalarized");
3528 }
3529 return obj;
3530 case Compile::SSC_always_false:
3531 if (from_inline || null_free) {
3532 if (!from_inline) {
3533 assert(safe_for_replace, "must be");
3534 null_check(obj);
3535 }
3536 // Inline type is null-free. Always throw an exception.
3537 builtin_throw(Deoptimization::Reason_class_check, makecon(TypeKlassPtr::make(klass)));
3538 return top();
3539 } else {
3540 // It needs a null check because a null will *pass* the cast check.
3541 const TypeOopPtr* objtp = _gvn.type(obj)->isa_oopptr();
3542 if (!objtp->maybe_null()) {
3543 builtin_throw(Deoptimization::Reason_class_check, makecon(TypeKlassPtr::make(objtp->klass())));
3544 return top();
3545 } else if (!too_many_traps_or_recompiles(Deoptimization::Reason_null_assert)) {
3546 return null_assert(obj);
3547 }
3548 break; // Fall through to full check
3549 }
3550 }
3551 }
3552 }
3553
3554 ciProfileData* data = NULL;
3555 if (failure_control == NULL) { // use MDO in regular case only
3556 assert(java_bc() == Bytecodes::_aastore ||
3557 java_bc() == Bytecodes::_checkcast,
3558 "interpreter profiles type checks only for these BCs");
3559 if (method()->method_data()->is_mature()) {
3560 data = method()->method_data()->bci_to_data(bci());
3561 }
3562 }
3563
3564 // Make the merge point
3565 enum { _obj_path = 1, _null_path, PATH_LIMIT };
3566 RegionNode* region = new RegionNode(PATH_LIMIT);
3567 Node* phi = new PhiNode(region, toop);
3568 _gvn.set_type(region, Type::CONTROL);
3569 _gvn.set_type(phi, toop);
3570
3571 C->set_has_split_ifs(true); // Has chance for split-if optimization
3572
3573 // Use null-cast information if it is available
3574 bool speculative_not_null = false;
3575 bool never_see_null = ((failure_control == NULL) // regular case only
3576 && seems_never_null(obj, data, speculative_not_null));
3577
3578 // Null check; get casted pointer; set region slot 3
3579 Node* null_ctl = top();
3580 Node* not_null_obj = NULL;
3581 if (from_inline) {
3582 not_null_obj = obj;
3583 } else if (null_free) {
3584 assert(safe_for_replace, "must be");
3585 not_null_obj = null_check(obj);
3586 } else {
3587 not_null_obj = null_check_oop(obj, &null_ctl, never_see_null, safe_for_replace, speculative_not_null);
3588 }
3589
3590 // If not_null_obj is dead, only null-path is taken
3591 if (stopped()) { // Doing instance-of on a NULL?
3592 set_control(null_ctl);
3593 return null();
3594 }
3595 region->init_req(_null_path, null_ctl);
3596 phi ->init_req(_null_path, null()); // Set null path value
3597 if (null_ctl == top()) {
3598 // Do this eagerly, so that pattern matches like is_diamond_phi
3599 // will work even during parsing.
3600 assert(_null_path == PATH_LIMIT-1, "delete last");
3601 region->del_req(_null_path);
3602 phi ->del_req(_null_path);
3603 }
3604
3605 Node* cast_obj = NULL;
3606 if (!from_inline && tk->klass_is_exact()) {
3607 // The following optimization tries to statically cast the speculative type of the object
3608 // (for example obtained during profiling) to the type of the superklass and then do a
3609 // dynamic check that the type of the object is what we expect. To work correctly
3610 // for checkcast and aastore the type of superklass should be exact.
3611 const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr();
3612 // We may not have profiling here or it may not help us. If we have
3613 // a speculative type use it to perform an exact cast.
3614 ciKlass* spec_obj_type = obj_type->speculative_type();
3615 if (spec_obj_type != NULL || data != NULL) {
3616 cast_obj = maybe_cast_profiled_receiver(not_null_obj, tk->klass(), spec_obj_type, safe_for_replace);
3617 if (cast_obj != NULL && cast_obj->is_InlineType()) {
3618 if (null_ctl != top()) {
3619 cast_obj = NULL; // A value that's sometimes null is not something we can optimize well
3620 } else {
3621 return cast_obj;
3622 }
3623 }
3624 if (cast_obj != NULL) {
3625 if (failure_control != NULL) // failure is now impossible
3626 (*failure_control) = top();
3627 // adjust the type of the phi to the exact klass:
3628 phi->raise_bottom_type(_gvn.type(cast_obj)->meet_speculative(TypePtr::NULL_PTR));
3629 }
3630 }
3631 }
3632
3633 if (cast_obj == NULL) {
3634 // Generate the subtype check
3635 Node* not_subtype_ctrl = gen_subtype_check(not_null_obj, superklass);
3636
3637 // Plug in success path into the merge
3638 cast_obj = from_inline ? not_null_obj : _gvn.transform(new CheckCastPPNode(control(), not_null_obj, toop));
3639 // Failure path ends in uncommon trap (or may be dead - failure impossible)
3640 if (failure_control == NULL) {
3641 if (not_subtype_ctrl != top()) { // If failure is possible
3642 PreserveJVMState pjvms(this);
3643 set_control(not_subtype_ctrl);
3644 Node* obj_klass = NULL;
3645 if (not_null_obj->is_InlineTypeBase()) {
3646 obj_klass = makecon(TypeKlassPtr::make(_gvn.type(not_null_obj)->inline_klass()));
3647 } else {
3648 obj_klass = load_object_klass(not_null_obj);
3649 }
3650 builtin_throw(Deoptimization::Reason_class_check, obj_klass);
3651 }
3652 } else {
3653 (*failure_control) = not_subtype_ctrl;
3654 }
3655 }
3656
3657 region->init_req(_obj_path, control());
3658 phi ->init_req(_obj_path, cast_obj);
3659
3660 // A merge of NULL or Casted-NotNull obj
3661 Node* res = _gvn.transform(phi);
3662
3663 // Note I do NOT always 'replace_in_map(obj,result)' here.
3664 // if( tk->klass()->can_be_primary_super() )
3665 // This means that if I successfully store an Object into an array-of-String
3666 // I 'forget' that the Object is really now known to be a String. I have to
3667 // do this because we don't have true union types for interfaces - if I store
3668 // a Baz into an array-of-Interface and then tell the optimizer it's an
3669 // Interface, I forget that it's also a Baz and cannot do Baz-like field
3670 // references to it. FIX THIS WHEN UNION TYPES APPEAR!
3671 // replace_in_map( obj, res );
3672
3673 // Return final merged results
3674 set_control( _gvn.transform(region) );
3675 record_for_igvn(region);
3676
3677 bool not_inline = !toop->can_be_inline_type();
3678 bool not_flattened = !UseFlatArray || not_inline || (toop->is_inlinetypeptr() && !toop->inline_klass()->flatten_array());
3679 if (EnableValhalla && not_flattened) {
3680 // Check if obj has been loaded from an array
3681 obj = obj->isa_DecodeN() ? obj->in(1) : obj;
3682 Node* array = NULL;
3683 if (obj->isa_Load()) {
3684 Node* address = obj->in(MemNode::Address);
3685 if (address->isa_AddP()) {
3686 array = address->as_AddP()->in(AddPNode::Base);
3687 }
3688 } else if (obj->is_Phi()) {
3689 Node* region = obj->in(0);
3690 // TODO make this more robust (see JDK-8231346)
3691 if (region->req() == 3 && region->in(2) != NULL && region->in(2)->in(0) != NULL) {
3692 IfNode* iff = region->in(2)->in(0)->isa_If();
3693 if (iff != NULL) {
3694 iff->is_flat_array_check(&_gvn, &array);
3695 }
3696 }
3697 }
3698 if (array != NULL) {
3699 const TypeAryPtr* ary_t = _gvn.type(array)->isa_aryptr();
3700 if (ary_t != NULL) {
3701 if (!ary_t->is_not_null_free() && not_inline) {
3702 // Casting array element to a non-inline-type, mark array as not null-free.
3703 Node* cast = _gvn.transform(new CheckCastPPNode(control(), array, ary_t->cast_to_not_null_free()));
3704 replace_in_map(array, cast);
3705 } else if (!ary_t->is_not_flat()) {
3706 // Casting array element to a non-flattened type, mark array as not flat.
3707 Node* cast = _gvn.transform(new CheckCastPPNode(control(), array, ary_t->cast_to_not_flat()));
3708 replace_in_map(array, cast);
3709 }
3710 }
3711 }
3712 }
3713
3714 if (!stopped() && !res->is_InlineTypeBase()) {
3715 res = record_profiled_receiver_for_speculation(res);
3716 if (toop->is_inlinetypeptr()) {
3717 Node* vt = InlineTypeNode::make_from_oop(this, res, toop->inline_klass(), !gvn().type(res)->maybe_null());
3718 res = vt;
3719 if (safe_for_replace) {
3720 if (vt->isa_InlineType() && C->inlining_incrementally()) {
3721 vt = vt->as_InlineType()->as_ptr(&_gvn);
3722 }
3723 replace_in_map(obj, vt);
3724 replace_in_map(not_null_obj, vt);
3725 replace_in_map(res, vt);
3726 }
3727 }
3728 }
3729 return res;
3730 }
3731
3732 Node* GraphKit::inline_type_test(Node* obj, bool is_inline) {
3733 Node* mark_adr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
3734 Node* mark = make_load(NULL, mark_adr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered);
3735 Node* mask = MakeConX(markWord::inline_type_pattern);
3736 Node* masked = _gvn.transform(new AndXNode(mark, mask));
3737 Node* cmp = _gvn.transform(new CmpXNode(masked, mask));
3738 return _gvn.transform(new BoolNode(cmp, is_inline ? BoolTest::eq : BoolTest::ne));
3739 }
3740
3741 Node* GraphKit::is_val_mirror(Node* mirror) {
3742 Node* p = basic_plus_adr(mirror, java_lang_Class::secondary_mirror_offset());
3743 Node* secondary_mirror = access_load_at(mirror, p, _gvn.type(p)->is_ptr(), TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR), T_OBJECT, IN_HEAP);
3744 Node* cmp = _gvn.transform(new CmpPNode(mirror, secondary_mirror));
3745 return _gvn.transform(new BoolNode(cmp, BoolTest::eq));
3746 }
3747
3748 Node* GraphKit::array_lh_test(Node* klass, jint mask, jint val, bool eq) {
3749 Node* lh_adr = basic_plus_adr(klass, in_bytes(Klass::layout_helper_offset()));
3750 // Make sure to use immutable memory here to enable hoisting the check out of loops
3751 Node* lh_val = _gvn.transform(LoadNode::make(_gvn, NULL, immutable_memory(), lh_adr, lh_adr->bottom_type()->is_ptr(), TypeInt::INT, T_INT, MemNode::unordered));
3752 Node* masked = _gvn.transform(new AndINode(lh_val, intcon(mask)));
3753 Node* cmp = _gvn.transform(new CmpINode(masked, intcon(val)));
3754 return _gvn.transform(new BoolNode(cmp, eq ? BoolTest::eq : BoolTest::ne));
3755 }
3756
3757 Node* GraphKit::flat_array_test(Node* ary, bool flat) {
3758 // We can't use immutable memory here because the mark word is mutable.
3759 // PhaseIdealLoop::move_flat_array_check_out_of_loop will make sure the
3760 // check is moved out of loops (mainly to enable loop unswitching).
3761 Node* mem = UseArrayMarkWordCheck ? memory(Compile::AliasIdxRaw) : immutable_memory();
3762 Node* cmp = _gvn.transform(new FlatArrayCheckNode(C, mem, ary));
3763 record_for_igvn(cmp); // Give it a chance to be optimized out by IGVN
3764 return _gvn.transform(new BoolNode(cmp, flat ? BoolTest::eq : BoolTest::ne));
3765 }
3766
3767 Node* GraphKit::null_free_array_test(Node* klass, bool null_free) {
3768 return array_lh_test(klass, Klass::_lh_null_free_bit_inplace, 0, !null_free);
3769 }
3770
3771 // Deoptimize if 'ary' is a null-free inline type array and 'val' is null
3772 Node* GraphKit::inline_array_null_guard(Node* ary, Node* val, int nargs, bool safe_for_replace) {
3773 const Type* val_t = _gvn.type(val);
3774 if (val->is_InlineType() || !TypePtr::NULL_PTR->higher_equal(val_t)) {
3775 return ary; // Never null
3776 }
3777 RegionNode* region = new RegionNode(3);
3778 Node* null_ctl = top();
3779 null_check_oop(val, &null_ctl);
3780 if (null_ctl != top()) {
3781 PreserveJVMState pjvms(this);
3782 set_control(null_ctl);
3783 {
3784 // Deoptimize if null-free array
3785 BuildCutout unless(this, null_free_array_test(load_object_klass(ary), /* null_free = */ false), PROB_MAX);
3786 inc_sp(nargs);
3787 uncommon_trap(Deoptimization::Reason_null_check,
3788 Deoptimization::Action_none);
3789 }
3790 region->init_req(1, control());
3791 }
3792 region->init_req(2, control());
3793 set_control(_gvn.transform(region));
3794 record_for_igvn(region);
3795 if (val_t == TypePtr::NULL_PTR) {
3796 // Since we were just successfully storing null, the array can't be null free.
3797 const TypeAryPtr* ary_t = _gvn.type(ary)->is_aryptr();
3798 ary_t = ary_t->cast_to_not_null_free();
3799 Node* cast = _gvn.transform(new CheckCastPPNode(control(), ary, ary_t));
3800 if (safe_for_replace) {
3801 replace_in_map(ary, cast);
3802 }
3803 ary = cast;
3804 }
3805 return ary;
3806 }
3807
3808 //------------------------------next_monitor-----------------------------------
3809 // What number should be given to the next monitor?
3810 int GraphKit::next_monitor() {
3811 int current = jvms()->monitor_depth()* C->sync_stack_slots();
3812 int next = current + C->sync_stack_slots();
3813 // Keep the toplevel high water mark current:
3814 if (C->fixed_slots() < next) C->set_fixed_slots(next);
3815 return current;
3816 }
3817
3818 //------------------------------insert_mem_bar---------------------------------
3819 // Memory barrier to avoid floating things around
3820 // The membar serves as a pinch point between both control and all memory slices.
3821 Node* GraphKit::insert_mem_bar(int opcode, Node* precedent) {
3822 MemBarNode* mb = MemBarNode::make(C, opcode, Compile::AliasIdxBot, precedent);
3823 mb->init_req(TypeFunc::Control, control());
3824 mb->init_req(TypeFunc::Memory, reset_memory());
3825 Node* membar = _gvn.transform(mb);
3853 }
3854 Node* membar = _gvn.transform(mb);
3855 set_control(_gvn.transform(new ProjNode(membar, TypeFunc::Control)));
3856 if (alias_idx == Compile::AliasIdxBot) {
3857 merged_memory()->set_base_memory(_gvn.transform(new ProjNode(membar, TypeFunc::Memory)));
3858 } else {
3859 set_memory(_gvn.transform(new ProjNode(membar, TypeFunc::Memory)),alias_idx);
3860 }
3861 return membar;
3862 }
3863
3864 //------------------------------shared_lock------------------------------------
3865 // Emit locking code.
3866 FastLockNode* GraphKit::shared_lock(Node* obj) {
3867 // bci is either a monitorenter bc or InvocationEntryBci
3868 // %%% SynchronizationEntryBCI is redundant; use InvocationEntryBci in interfaces
3869 assert(SynchronizationEntryBCI == InvocationEntryBci, "");
3870
3871 if( !GenerateSynchronizationCode )
3872 return NULL; // Not locking things?
3873
3874 if (stopped()) // Dead monitor?
3875 return NULL;
3876
3877 assert(dead_locals_are_killed(), "should kill locals before sync. point");
3878
3879 // Box the stack location
3880 Node* box = _gvn.transform(new BoxLockNode(next_monitor()));
3881 Node* mem = reset_memory();
3882
3883 FastLockNode * flock = _gvn.transform(new FastLockNode(0, obj, box) )->as_FastLock();
3884
3885 // Create the rtm counters for this fast lock if needed.
3886 flock->create_rtm_lock_counter(sync_jvms()); // sync_jvms used to get current bci
3887
3888 // Add monitor to debug info for the slow path. If we block inside the
3889 // slow path and de-opt, we need the monitor hanging around
3890 map()->push_monitor( flock );
3891
3892 const TypeFunc *tf = LockNode::lock_type();
3893 LockNode *lock = new LockNode(C, tf);
3922 }
3923 #endif
3924
3925 return flock;
3926 }
3927
3928
3929 //------------------------------shared_unlock----------------------------------
3930 // Emit unlocking code.
3931 void GraphKit::shared_unlock(Node* box, Node* obj) {
3932 // bci is either a monitorenter bc or InvocationEntryBci
3933 // %%% SynchronizationEntryBCI is redundant; use InvocationEntryBci in interfaces
3934 assert(SynchronizationEntryBCI == InvocationEntryBci, "");
3935
3936 if( !GenerateSynchronizationCode )
3937 return;
3938 if (stopped()) { // Dead monitor?
3939 map()->pop_monitor(); // Kill monitor from debug info
3940 return;
3941 }
3942 assert(!obj->is_InlineTypeBase(), "should not unlock on inline type");
3943
3944 // Memory barrier to avoid floating things down past the locked region
3945 insert_mem_bar(Op_MemBarReleaseLock);
3946
3947 const TypeFunc *tf = OptoRuntime::complete_monitor_exit_Type();
3948 UnlockNode *unlock = new UnlockNode(C, tf);
3949 #ifdef ASSERT
3950 unlock->set_dbg_jvms(sync_jvms());
3951 #endif
3952 uint raw_idx = Compile::AliasIdxRaw;
3953 unlock->init_req( TypeFunc::Control, control() );
3954 unlock->init_req( TypeFunc::Memory , memory(raw_idx) );
3955 unlock->init_req( TypeFunc::I_O , top() ) ; // does no i/o
3956 unlock->init_req( TypeFunc::FramePtr, frameptr() );
3957 unlock->init_req( TypeFunc::ReturnAdr, top() );
3958
3959 unlock->init_req(TypeFunc::Parms + 0, obj);
3960 unlock->init_req(TypeFunc::Parms + 1, box);
3961 unlock = _gvn.transform(unlock)->as_Unlock();
3962
3963 Node* mem = reset_memory();
3964
3965 // unlock has no side-effects, sets few values
3966 set_predefined_output_for_runtime_call(unlock, mem, TypeRawPtr::BOTTOM);
3967
3968 // Kill monitor from debug info
3969 map()->pop_monitor( );
3970 }
3971
3972 //-------------------------------get_layout_helper-----------------------------
3973 // If the given klass is a constant or known to be an array,
3974 // fetch the constant layout helper value into constant_value
3975 // and return (Node*)NULL. Otherwise, load the non-constant
3976 // layout helper value, and return the node which represents it.
3977 // This two-faced routine is useful because allocation sites
3978 // almost always feature constant types.
3979 Node* GraphKit::get_layout_helper(Node* klass_node, jint& constant_value) {
3980 const TypeKlassPtr* inst_klass = _gvn.type(klass_node)->isa_klassptr();
3981 if (!StressReflectiveCode && inst_klass != NULL) {
3982 ciKlass* klass = inst_klass->klass();
3983 assert(klass != NULL, "klass should not be NULL");
3984 bool xklass = inst_klass->klass_is_exact();
3985 bool can_be_flattened = false;
3986 if (UseFlatArray && klass->is_obj_array_klass() && !klass->as_obj_array_klass()->is_elem_null_free()) {
3987 // The runtime type of [LMyValue might be [QMyValue due to [QMyValue <: [LMyValue.
3988 ciKlass* elem = klass->as_obj_array_klass()->element_klass();
3989 can_be_flattened = elem->can_be_inline_klass() && (!elem->is_inlinetype() || elem->flatten_array());
3990 }
3991 if (!can_be_flattened && (xklass || klass->is_array_klass())) {
3992 jint lhelper = klass->layout_helper();
3993 if (lhelper != Klass::_lh_neutral_value) {
3994 constant_value = lhelper;
3995 return (Node*) NULL;
3996 }
3997 }
3998 }
3999 constant_value = Klass::_lh_neutral_value; // put in a known value
4000 Node* lhp = basic_plus_adr(klass_node, klass_node, in_bytes(Klass::layout_helper_offset()));
4001 return make_load(NULL, lhp, TypeInt::INT, T_INT, MemNode::unordered);
4002 }
4003
4004 // We just put in an allocate/initialize with a big raw-memory effect.
4005 // Hook selected additional alias categories on the initialization.
4006 static void hook_memory_on_init(GraphKit& kit, int alias_idx,
4007 MergeMemNode* init_in_merge,
4008 Node* init_out_raw) {
4009 DEBUG_ONLY(Node* init_in_raw = init_in_merge->base_memory());
4010 assert(init_in_merge->memory_at(alias_idx) == init_in_raw, "");
4011
4012 Node* prevmem = kit.memory(alias_idx);
4013 init_in_merge->set_memory_at(alias_idx, prevmem);
4014 if (init_out_raw != NULL) {
4015 kit.set_memory(init_out_raw, alias_idx);
4016 }
4017 }
4018
4019 //---------------------------set_output_for_allocation-------------------------
4020 Node* GraphKit::set_output_for_allocation(AllocateNode* alloc,
4021 const TypeOopPtr* oop_type,
4022 bool deoptimize_on_exception) {
4023 int rawidx = Compile::AliasIdxRaw;
4024 alloc->set_req( TypeFunc::FramePtr, frameptr() );
4025 add_safepoint_edges(alloc);
4026 Node* allocx = _gvn.transform(alloc);
4027 set_control( _gvn.transform(new ProjNode(allocx, TypeFunc::Control) ) );
4028 // create memory projection for i_o
4029 set_memory ( _gvn.transform( new ProjNode(allocx, TypeFunc::Memory, true) ), rawidx );
4030 make_slow_call_ex(allocx, env()->Throwable_klass(), true, deoptimize_on_exception);
4031
4032 // create a memory projection as for the normal control path
4033 Node* malloc = _gvn.transform(new ProjNode(allocx, TypeFunc::Memory));
4034 set_memory(malloc, rawidx);
4035
4036 // a normal slow-call doesn't change i_o, but an allocation does
4037 // we create a separate i_o projection for the normal control path
4038 set_i_o(_gvn.transform( new ProjNode(allocx, TypeFunc::I_O, false) ) );
4039 Node* rawoop = _gvn.transform( new ProjNode(allocx, TypeFunc::Parms) );
4040
4041 // put in an initialization barrier
4042 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, rawidx,
4043 rawoop)->as_Initialize();
4044 assert(alloc->initialization() == init, "2-way macro link must work");
4045 assert(init ->allocation() == alloc, "2-way macro link must work");
4046 {
4047 // Extract memory strands which may participate in the new object's
4048 // initialization, and source them from the new InitializeNode.
4049 // This will allow us to observe initializations when they occur,
4050 // and link them properly (as a group) to the InitializeNode.
4051 assert(init->in(InitializeNode::Memory) == malloc, "");
4052 MergeMemNode* minit_in = MergeMemNode::make(malloc);
4053 init->set_req(InitializeNode::Memory, minit_in);
4054 record_for_igvn(minit_in); // fold it up later, if possible
4055 _gvn.set_type(minit_in, Type::MEMORY);
4056 Node* minit_out = memory(rawidx);
4057 assert(minit_out->is_Proj() && minit_out->in(0) == init, "");
4058 // Add an edge in the MergeMem for the header fields so an access
4059 // to one of those has correct memory state
4060 set_memory(minit_out, C->get_alias_index(oop_type->add_offset(oopDesc::mark_offset_in_bytes())));
4061 set_memory(minit_out, C->get_alias_index(oop_type->add_offset(oopDesc::klass_offset_in_bytes())));
4062 if (oop_type->isa_aryptr()) {
4063 const TypeAryPtr* arytype = oop_type->is_aryptr();
4064 if (arytype->klass()->is_flat_array_klass()) {
4065 // Initially all flattened array accesses share a single slice
4066 // but that changes after parsing. Prepare the memory graph so
4067 // it can optimize flattened array accesses properly once they
4068 // don't share a single slice.
4069 assert(C->flattened_accesses_share_alias(), "should be set at parse time");
4070 C->set_flattened_accesses_share_alias(false);
4071 ciFlatArrayKlass* vak = arytype->klass()->as_flat_array_klass();
4072 ciInlineKlass* vk = vak->element_klass()->as_inline_klass();
4073 for (int i = 0, len = vk->nof_nonstatic_fields(); i < len; i++) {
4074 ciField* field = vk->nonstatic_field_at(i);
4075 if (field->offset() >= TrackedInitializationLimit * HeapWordSize)
4076 continue; // do not bother to track really large numbers of fields
4077 int off_in_vt = field->offset() - vk->first_field_offset();
4078 const TypePtr* adr_type = arytype->with_field_offset(off_in_vt)->add_offset(Type::OffsetBot);
4079 int fieldidx = C->get_alias_index(adr_type, true);
4080 // Pass NULL for init_out. Having per flat array element field memory edges as uses of the Initialize node
4081 // can result in per flat array field Phis to be created which confuses the logic of
4082 // Compile::adjust_flattened_array_access_aliases().
4083 hook_memory_on_init(*this, fieldidx, minit_in, NULL);
4084 }
4085 C->set_flattened_accesses_share_alias(true);
4086 hook_memory_on_init(*this, C->get_alias_index(TypeAryPtr::INLINES), minit_in, minit_out);
4087 } else {
4088 const TypePtr* telemref = oop_type->add_offset(Type::OffsetBot);
4089 int elemidx = C->get_alias_index(telemref);
4090 hook_memory_on_init(*this, elemidx, minit_in, minit_out);
4091 }
4092 } else if (oop_type->isa_instptr()) {
4093 set_memory(minit_out, C->get_alias_index(oop_type)); // mark word
4094 ciInstanceKlass* ik = oop_type->klass()->as_instance_klass();
4095 for (int i = 0, len = ik->nof_nonstatic_fields(); i < len; i++) {
4096 ciField* field = ik->nonstatic_field_at(i);
4097 if (field->offset() >= TrackedInitializationLimit * HeapWordSize)
4098 continue; // do not bother to track really large numbers of fields
4099 // Find (or create) the alias category for this field:
4100 int fieldidx = C->alias_type(field)->index();
4101 hook_memory_on_init(*this, fieldidx, minit_in, minit_out);
4102 }
4103 }
4104 }
4105
4106 // Cast raw oop to the real thing...
4107 Node* javaoop = new CheckCastPPNode(control(), rawoop, oop_type);
4108 javaoop = _gvn.transform(javaoop);
4109 C->set_recent_alloc(control(), javaoop);
4110 assert(just_allocated_object(control()) == javaoop, "just allocated");
4111
4112 #ifdef ASSERT
4113 { // Verify that the AllocateNode::Ideal_allocation recognizers work:
4124 assert(alloc->in(AllocateNode::ALength)->is_top(), "no length, please");
4125 }
4126 }
4127 #endif //ASSERT
4128
4129 return javaoop;
4130 }
4131
4132 //---------------------------new_instance--------------------------------------
4133 // This routine takes a klass_node which may be constant (for a static type)
4134 // or may be non-constant (for reflective code). It will work equally well
4135 // for either, and the graph will fold nicely if the optimizer later reduces
4136 // the type to a constant.
4137 // The optional arguments are for specialized use by intrinsics:
4138 // - If 'extra_slow_test' if not null is an extra condition for the slow-path.
4139 // - If 'return_size_val', report the the total object size to the caller.
4140 // - deoptimize_on_exception controls how Java exceptions are handled (rethrow vs deoptimize)
4141 Node* GraphKit::new_instance(Node* klass_node,
4142 Node* extra_slow_test,
4143 Node* *return_size_val,
4144 bool deoptimize_on_exception,
4145 InlineTypeBaseNode* inline_type_node) {
4146 // Compute size in doublewords
4147 // The size is always an integral number of doublewords, represented
4148 // as a positive bytewise size stored in the klass's layout_helper.
4149 // The layout_helper also encodes (in a low bit) the need for a slow path.
4150 jint layout_con = Klass::_lh_neutral_value;
4151 Node* layout_val = get_layout_helper(klass_node, layout_con);
4152 bool layout_is_con = (layout_val == NULL);
4153
4154 if (extra_slow_test == NULL) extra_slow_test = intcon(0);
4155 // Generate the initial go-slow test. It's either ALWAYS (return a
4156 // Node for 1) or NEVER (return a NULL) or perhaps (in the reflective
4157 // case) a computed value derived from the layout_helper.
4158 Node* initial_slow_test = NULL;
4159 if (layout_is_con) {
4160 assert(!StressReflectiveCode, "stress mode does not use these paths");
4161 bool must_go_slow = Klass::layout_helper_needs_slow_path(layout_con);
4162 initial_slow_test = must_go_slow ? intcon(1) : extra_slow_test;
4163 } else { // reflective case
4164 // This reflective path is used by Unsafe.allocateInstance.
4165 // (It may be stress-tested by specifying StressReflectiveCode.)
4166 // Basically, we want to get into the VM is there's an illegal argument.
4167 Node* bit = intcon(Klass::_lh_instance_slow_path_bit);
4168 initial_slow_test = _gvn.transform( new AndINode(layout_val, bit) );
4169 if (extra_slow_test != intcon(0)) {
4170 initial_slow_test = _gvn.transform( new OrINode(initial_slow_test, extra_slow_test) );
4171 }
4172 // (Macro-expander will further convert this to a Bool, if necessary.)
4183
4184 // Clear the low bits to extract layout_helper_size_in_bytes:
4185 assert((int)Klass::_lh_instance_slow_path_bit < BytesPerLong, "clear bit");
4186 Node* mask = MakeConX(~ (intptr_t)right_n_bits(LogBytesPerLong));
4187 size = _gvn.transform( new AndXNode(size, mask) );
4188 }
4189 if (return_size_val != NULL) {
4190 (*return_size_val) = size;
4191 }
4192
4193 // This is a precise notnull oop of the klass.
4194 // (Actually, it need not be precise if this is a reflective allocation.)
4195 // It's what we cast the result to.
4196 const TypeKlassPtr* tklass = _gvn.type(klass_node)->isa_klassptr();
4197 if (!tklass) tklass = TypeInstKlassPtr::OBJECT;
4198 const TypeOopPtr* oop_type = tklass->as_instance_type();
4199
4200 // Now generate allocation code
4201
4202 // The entire memory state is needed for slow path of the allocation
4203 // since GC and deoptimization can happen.
4204 Node *mem = reset_memory();
4205 set_all_memory(mem); // Create new memory state
4206
4207 AllocateNode* alloc = new AllocateNode(C, AllocateNode::alloc_type(Type::TOP),
4208 control(), mem, i_o(),
4209 size, klass_node,
4210 initial_slow_test, inline_type_node);
4211
4212 return set_output_for_allocation(alloc, oop_type, deoptimize_on_exception);
4213 }
4214
4215 //-------------------------------new_array-------------------------------------
4216 // helper for newarray and anewarray
4217 // The 'length' parameter is (obviously) the length of the array.
4218 // See comments on new_instance for the meaning of the other arguments.
4219 Node* GraphKit::new_array(Node* klass_node, // array klass (maybe variable)
4220 Node* length, // number of array elements
4221 int nargs, // number of arguments to push back for uncommon trap
4222 Node* *return_size_val,
4223 bool deoptimize_on_exception) {
4224 jint layout_con = Klass::_lh_neutral_value;
4225 Node* layout_val = get_layout_helper(klass_node, layout_con);
4226 bool layout_is_con = (layout_val == NULL);
4227
4228 if (!layout_is_con && !StressReflectiveCode &&
4229 !too_many_traps(Deoptimization::Reason_class_check)) {
4230 // This is a reflective array creation site.
4231 // Optimistically assume that it is a subtype of Object[],
4232 // so that we can fold up all the address arithmetic.
4233 layout_con = Klass::array_layout_helper(T_OBJECT);
4234 Node* cmp_lh = _gvn.transform( new CmpINode(layout_val, intcon(layout_con)) );
4235 Node* bol_lh = _gvn.transform( new BoolNode(cmp_lh, BoolTest::eq) );
4236 { BuildCutout unless(this, bol_lh, PROB_MAX);
4237 inc_sp(nargs);
4238 uncommon_trap(Deoptimization::Reason_class_check,
4239 Deoptimization::Action_maybe_recompile);
4240 }
4241 layout_val = NULL;
4242 layout_is_con = true;
4243 }
4244
4245 // Generate the initial go-slow test. Make sure we do not overflow
4246 // if length is huge (near 2Gig) or negative! We do not need
4247 // exact double-words here, just a close approximation of needed
4248 // double-words. We can't add any offset or rounding bits, lest we
4249 // take a size -1 of bytes and make it positive. Use an unsigned
4250 // compare, so negative sizes look hugely positive.
4251 int fast_size_limit = FastAllocateSizeLimit;
4252 if (layout_is_con) {
4253 assert(!StressReflectiveCode, "stress mode does not use these paths");
4254 // Increase the size limit if we have exact knowledge of array type.
4255 int log2_esize = Klass::layout_helper_log2_element_size(layout_con);
4256 fast_size_limit <<= MAX2(LogBytesPerLong - log2_esize, 0);
4257 }
4258
4259 Node* initial_slow_cmp = _gvn.transform( new CmpUNode( length, intcon( fast_size_limit ) ) );
4260 Node* initial_slow_test = _gvn.transform( new BoolNode( initial_slow_cmp, BoolTest::gt ) );
4261
4262 // --- Size Computation ---
4263 // array_size = round_to_heap(array_header + (length << elem_shift));
4264 // where round_to_heap(x) == align_to(x, MinObjAlignmentInBytes)
4265 // and align_to(x, y) == ((x + y-1) & ~(y-1))
4266 // The rounding mask is strength-reduced, if possible.
4267 int round_mask = MinObjAlignmentInBytes - 1;
4268 Node* header_size = NULL;
4269 int header_size_min = arrayOopDesc::base_offset_in_bytes(T_BYTE);
4270 // (T_BYTE has the weakest alignment and size restrictions...)
4271 if (layout_is_con) {
4272 int hsize = Klass::layout_helper_header_size(layout_con);
4273 int eshift = Klass::layout_helper_log2_element_size(layout_con);
4274 bool is_flat_array = Klass::layout_helper_is_flatArray(layout_con);
4275 if ((round_mask & ~right_n_bits(eshift)) == 0)
4276 round_mask = 0; // strength-reduce it if it goes away completely
4277 assert(is_flat_array || (hsize & right_n_bits(eshift)) == 0, "hsize is pre-rounded");
4278 assert(header_size_min <= hsize, "generic minimum is smallest");
4279 header_size_min = hsize;
4280 header_size = intcon(hsize + round_mask);
4281 } else {
4282 Node* hss = intcon(Klass::_lh_header_size_shift);
4283 Node* hsm = intcon(Klass::_lh_header_size_mask);
4284 Node* hsize = _gvn.transform( new URShiftINode(layout_val, hss) );
4285 hsize = _gvn.transform( new AndINode(hsize, hsm) );
4286 Node* mask = intcon(round_mask);
4287 header_size = _gvn.transform( new AddINode(hsize, mask) );
4288 }
4289
4290 Node* elem_shift = NULL;
4291 if (layout_is_con) {
4292 int eshift = Klass::layout_helper_log2_element_size(layout_con);
4293 if (eshift != 0)
4294 elem_shift = intcon(eshift);
4295 } else {
4296 // There is no need to mask or shift this value.
4297 // The semantics of LShiftINode include an implicit mask to 0x1F.
4341 // places, one where the length is sharply limited, and the other
4342 // after a successful allocation.
4343 Node* abody = lengthx;
4344 if (elem_shift != NULL)
4345 abody = _gvn.transform( new LShiftXNode(lengthx, elem_shift) );
4346 Node* size = _gvn.transform( new AddXNode(headerx, abody) );
4347 if (round_mask != 0) {
4348 Node* mask = MakeConX(~round_mask);
4349 size = _gvn.transform( new AndXNode(size, mask) );
4350 }
4351 // else if round_mask == 0, the size computation is self-rounding
4352
4353 if (return_size_val != NULL) {
4354 // This is the size
4355 (*return_size_val) = size;
4356 }
4357
4358 // Now generate allocation code
4359
4360 // The entire memory state is needed for slow path of the allocation
4361 // since GC and deoptimization can happen.
4362 Node *mem = reset_memory();
4363 set_all_memory(mem); // Create new memory state
4364
4365 if (initial_slow_test->is_Bool()) {
4366 // Hide it behind a CMoveI, or else PhaseIdealLoop::split_up will get sick.
4367 initial_slow_test = initial_slow_test->as_Bool()->as_int_value(&_gvn);
4368 }
4369
4370 const TypeKlassPtr* ary_klass = _gvn.type(klass_node)->isa_klassptr();
4371 const TypeOopPtr* ary_type = ary_klass->as_instance_type();
4372 const TypeAryPtr* ary_ptr = ary_type->isa_aryptr();
4373
4374 // Inline type array variants:
4375 // - null-ok: MyValue.ref[] (ciObjArrayKlass "[LMyValue")
4376 // - null-free: MyValue.val[] (ciObjArrayKlass "[QMyValue")
4377 // - null-free, flattened: MyValue.val[] (ciFlatArrayKlass "[QMyValue")
4378 // Check if array is a null-free, non-flattened inline type array
4379 // that needs to be initialized with the default inline type.
4380 Node* default_value = NULL;
4381 Node* raw_default_value = NULL;
4382 if (ary_ptr != NULL && ary_ptr->klass_is_exact()) {
4383 // Array type is known
4384 if (ary_ptr->klass()->as_array_klass()->is_elem_null_free()) {
4385 ciInlineKlass* vk = ary_ptr->klass()->as_array_klass()->element_klass()->as_inline_klass();
4386 if (!vk->flatten_array()) {
4387 default_value = InlineTypeNode::default_oop(gvn(), vk);
4388 }
4389 }
4390 } else if (ary_klass->klass()->can_be_inline_array_klass()) {
4391 // Array type is not known, add runtime checks
4392 assert(!ary_klass->klass_is_exact(), "unexpected exact type");
4393 Node* r = new RegionNode(3);
4394 default_value = new PhiNode(r, TypeInstPtr::BOTTOM);
4395
4396 Node* bol = array_lh_test(klass_node, Klass::_lh_array_tag_vt_value_bit_inplace | Klass::_lh_null_free_bit_inplace, Klass::_lh_null_free_bit_inplace);
4397 IfNode* iff = create_and_map_if(control(), bol, PROB_FAIR, COUNT_UNKNOWN);
4398
4399 // Null-free, non-flattened inline type array, initialize with the default value
4400 set_control(_gvn.transform(new IfTrueNode(iff)));
4401 Node* p = basic_plus_adr(klass_node, in_bytes(ArrayKlass::element_klass_offset()));
4402 Node* eklass = _gvn.transform(LoadKlassNode::make(_gvn, control(), immutable_memory(), p, TypeInstPtr::KLASS));
4403 Node* adr_fixed_block_addr = basic_plus_adr(eklass, in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset()));
4404 Node* adr_fixed_block = make_load(control(), adr_fixed_block_addr, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
4405 Node* default_value_offset_addr = basic_plus_adr(adr_fixed_block, in_bytes(InlineKlass::default_value_offset_offset()));
4406 Node* default_value_offset = make_load(control(), default_value_offset_addr, TypeInt::INT, T_INT, MemNode::unordered);
4407 Node* elem_mirror = load_mirror_from_klass(eklass);
4408 Node* default_value_addr = basic_plus_adr(elem_mirror, ConvI2X(default_value_offset));
4409 Node* val = access_load_at(elem_mirror, default_value_addr, _gvn.type(default_value_addr)->is_ptr(), TypeInstPtr::BOTTOM, T_OBJECT, IN_HEAP);
4410 r->init_req(1, control());
4411 default_value->init_req(1, val);
4412
4413 // Otherwise initialize with all zero
4414 r->init_req(2, _gvn.transform(new IfFalseNode(iff)));
4415 default_value->init_req(2, null());
4416
4417 set_control(_gvn.transform(r));
4418 default_value = _gvn.transform(default_value);
4419 }
4420 if (default_value != NULL) {
4421 if (UseCompressedOops) {
4422 // With compressed oops, the 64-bit init value is built from two 32-bit compressed oops
4423 default_value = _gvn.transform(new EncodePNode(default_value, default_value->bottom_type()->make_narrowoop()));
4424 Node* lower = _gvn.transform(new CastP2XNode(control(), default_value));
4425 Node* upper = _gvn.transform(new LShiftLNode(lower, intcon(32)));
4426 raw_default_value = _gvn.transform(new OrLNode(lower, upper));
4427 } else {
4428 raw_default_value = _gvn.transform(new CastP2XNode(control(), default_value));
4429 }
4430 }
4431
4432 // Create the AllocateArrayNode and its result projections
4433 AllocateArrayNode* alloc = new AllocateArrayNode(C, AllocateArrayNode::alloc_type(TypeInt::INT),
4434 control(), mem, i_o(),
4435 size, klass_node,
4436 initial_slow_test,
4437 length, default_value,
4438 raw_default_value);
4439
4440 // Cast to correct type. Note that the klass_node may be constant or not,
4441 // and in the latter case the actual array type will be inexact also.
4442 // (This happens via a non-constant argument to inline_native_newArray.)
4443 // In any case, the value of klass_node provides the desired array type.
4444 const TypeInt* length_type = _gvn.find_int_type(length);
4445 if (ary_type->isa_aryptr() && length_type != NULL) {
4446 // Try to get a better type than POS for the size
4447 ary_type = ary_type->is_aryptr()->cast_to_size(length_type);
4448 }
4449
4450 Node* javaoop = set_output_for_allocation(alloc, ary_type, deoptimize_on_exception);
4451
4452 array_ideal_length(alloc, ary_type, true);
4453 return javaoop;
4454 }
4455
4456 // The following "Ideal_foo" functions are placed here because they recognize
4457 // the graph shapes created by the functions immediately above.
4458
4459 //---------------------------Ideal_allocation----------------------------------
4460 // Given an oop pointer or raw pointer, see if it feeds from an AllocateNode.
4461 AllocateNode* AllocateNode::Ideal_allocation(Node* ptr, PhaseTransform* phase) {
4462 if (ptr == NULL) { // reduce dumb test in callers
4463 return NULL;
4464 }
4573 set_all_memory(ideal.merged_memory());
4574 set_i_o(ideal.i_o());
4575 set_control(ideal.ctrl());
4576 }
4577
4578 void GraphKit::final_sync(IdealKit& ideal) {
4579 // Final sync IdealKit and graphKit.
4580 sync_kit(ideal);
4581 }
4582
4583 Node* GraphKit::load_String_length(Node* str, bool set_ctrl) {
4584 Node* len = load_array_length(load_String_value(str, set_ctrl));
4585 Node* coder = load_String_coder(str, set_ctrl);
4586 // Divide length by 2 if coder is UTF16
4587 return _gvn.transform(new RShiftINode(len, coder));
4588 }
4589
4590 Node* GraphKit::load_String_value(Node* str, bool set_ctrl) {
4591 int value_offset = java_lang_String::value_offset();
4592 const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4593 false, NULL, Type::Offset(0));
4594 const TypePtr* value_field_type = string_type->add_offset(value_offset);
4595 const TypeAryPtr* value_type = TypeAryPtr::make(TypePtr::NotNull,
4596 TypeAry::make(TypeInt::BYTE, TypeInt::POS, false, true, true),
4597 ciTypeArrayKlass::make(T_BYTE), true, Type::Offset(0));
4598 Node* p = basic_plus_adr(str, str, value_offset);
4599 Node* load = access_load_at(str, p, value_field_type, value_type, T_OBJECT,
4600 IN_HEAP | (set_ctrl ? C2_CONTROL_DEPENDENT_LOAD : 0) | MO_UNORDERED);
4601 return load;
4602 }
4603
4604 Node* GraphKit::load_String_coder(Node* str, bool set_ctrl) {
4605 if (!CompactStrings) {
4606 return intcon(java_lang_String::CODER_UTF16);
4607 }
4608 int coder_offset = java_lang_String::coder_offset();
4609 const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4610 false, NULL, Type::Offset(0));
4611 const TypePtr* coder_field_type = string_type->add_offset(coder_offset);
4612
4613 Node* p = basic_plus_adr(str, str, coder_offset);
4614 Node* load = access_load_at(str, p, coder_field_type, TypeInt::BYTE, T_BYTE,
4615 IN_HEAP | (set_ctrl ? C2_CONTROL_DEPENDENT_LOAD : 0) | MO_UNORDERED);
4616 return load;
4617 }
4618
4619 void GraphKit::store_String_value(Node* str, Node* value) {
4620 int value_offset = java_lang_String::value_offset();
4621 const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4622 false, NULL, Type::Offset(0));
4623 const TypePtr* value_field_type = string_type->add_offset(value_offset);
4624
4625 access_store_at(str, basic_plus_adr(str, value_offset), value_field_type,
4626 value, TypeAryPtr::BYTES, T_OBJECT, IN_HEAP | MO_UNORDERED);
4627 }
4628
4629 void GraphKit::store_String_coder(Node* str, Node* value) {
4630 int coder_offset = java_lang_String::coder_offset();
4631 const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::NotNull, C->env()->String_klass(),
4632 false, NULL, Type::Offset(0));
4633 const TypePtr* coder_field_type = string_type->add_offset(coder_offset);
4634
4635 access_store_at(str, basic_plus_adr(str, coder_offset), coder_field_type,
4636 value, TypeInt::BYTE, T_BYTE, IN_HEAP | MO_UNORDERED);
4637 }
4638
4639 // Capture src and dst memory state with a MergeMemNode
4640 Node* GraphKit::capture_memory(const TypePtr* src_type, const TypePtr* dst_type) {
4641 if (src_type == dst_type) {
4642 // Types are equal, we don't need a MergeMemNode
4643 return memory(src_type);
4644 }
4645 MergeMemNode* merge = MergeMemNode::make(map()->memory());
4646 record_for_igvn(merge); // fold it up later, if possible
4647 int src_idx = C->get_alias_index(src_type);
4648 int dst_idx = C->get_alias_index(dst_type);
4649 merge->set_memory_at(src_idx, memory(src_idx));
4650 merge->set_memory_at(dst_idx, memory(dst_idx));
4651 return merge;
4652 }
4725 i_char->init_req(2, AddI(i_char, intcon(2)));
4726
4727 set_control(IfFalse(iff));
4728 set_memory(st, TypeAryPtr::BYTES);
4729 }
4730
4731 Node* GraphKit::make_constant_from_field(ciField* field, Node* obj) {
4732 if (!field->is_constant()) {
4733 return NULL; // Field not marked as constant.
4734 }
4735 ciInstance* holder = NULL;
4736 if (!field->is_static()) {
4737 ciObject* const_oop = obj->bottom_type()->is_oopptr()->const_oop();
4738 if (const_oop != NULL && const_oop->is_instance()) {
4739 holder = const_oop->as_instance();
4740 }
4741 }
4742 const Type* con_type = Type::make_constant_from_field(field, holder, field->layout_type(),
4743 /*is_unsigned_load=*/false);
4744 if (con_type != NULL) {
4745 Node* con = makecon(con_type);
4746 if (field->type()->is_inlinetype()) {
4747 con = InlineTypeNode::make_from_oop(this, con, field->type()->as_inline_klass(), field->is_null_free());
4748 } else if (con_type->is_inlinetypeptr()) {
4749 con = InlineTypeNode::make_from_oop(this, con, con_type->inline_klass(), field->is_null_free());
4750 }
4751 return con;
4752 }
4753 return NULL;
4754 }
4755
4756 //---------------------------load_mirror_from_klass----------------------------
4757 // Given a klass oop, load its java mirror (a java.lang.Class oop).
4758 Node* GraphKit::load_mirror_from_klass(Node* klass) {
4759 Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset()));
4760 Node* load = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
4761 // mirror = ((OopHandle)mirror)->resolve();
4762 return access_load(load, TypeInstPtr::MIRROR, T_OBJECT, IN_NATIVE);
4763 }
|