14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "c1/c1_Compilation.hpp"
27 #include "c1/c1_Defs.hpp"
28 #include "c1/c1_FrameMap.hpp"
29 #include "c1/c1_Instruction.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_LIRGenerator.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "ci/ciObjArray.hpp"
36 #include "ci/ciUtilities.hpp"
37 #include "gc/shared/barrierSet.hpp"
38 #include "gc/shared/c1/barrierSetC1.hpp"
39 #include "oops/klass.inline.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #include "runtime/stubRoutines.hpp"
42 #include "runtime/vm_version.hpp"
43 #include "utilities/bitMap.inline.hpp"
44 #include "utilities/macros.hpp"
45 #include "utilities/powerOfTwo.hpp"
46
47 #ifdef ASSERT
48 #define __ gen()->lir(__FILE__, __LINE__)->
49 #else
50 #define __ gen()->lir()->
51 #endif
52
53 #ifndef PATCHED_ADDR
193 ResolveNode* source = source_node(src);
194 source->append(destination_node(dest));
195 }
196
197
198 //--------------------------------------------------------------
199 // LIRItem
200
201 void LIRItem::set_result(LIR_Opr opr) {
202 assert(value()->operand()->is_illegal() || value()->operand()->is_constant(), "operand should never change");
203 value()->set_operand(opr);
204
205 if (opr->is_virtual()) {
206 _gen->_instruction_for_operand.at_put_grow(opr->vreg_number(), value(), NULL);
207 }
208
209 _result = opr;
210 }
211
212 void LIRItem::load_item() {
213 if (result()->is_illegal()) {
214 // update the items result
215 _result = value()->operand();
216 }
217 if (!result()->is_register()) {
218 LIR_Opr reg = _gen->new_register(value()->type());
219 __ move(result(), reg);
220 if (result()->is_constant()) {
221 _result = reg;
222 } else {
223 set_result(reg);
224 }
225 }
226 }
227
228
229 void LIRItem::load_for_store(BasicType type) {
230 if (_gen->can_store_as_constant(value(), type)) {
231 _result = value()->operand();
232 if (!_result->is_constant()) {
588 assert(right_op != result_op, "malformed");
589 __ move(left_op, result_op);
590 left_op = result_op;
591 }
592
593 switch(code) {
594 case Bytecodes::_iand:
595 case Bytecodes::_land: __ logical_and(left_op, right_op, result_op); break;
596
597 case Bytecodes::_ior:
598 case Bytecodes::_lor: __ logical_or(left_op, right_op, result_op); break;
599
600 case Bytecodes::_ixor:
601 case Bytecodes::_lxor: __ logical_xor(left_op, right_op, result_op); break;
602
603 default: ShouldNotReachHere();
604 }
605 }
606
607
608 void LIRGenerator::monitor_enter(LIR_Opr object, LIR_Opr lock, LIR_Opr hdr, LIR_Opr scratch, int monitor_no, CodeEmitInfo* info_for_exception, CodeEmitInfo* info) {
609 if (!GenerateSynchronizationCode) return;
610 // for slow path, use debug info for state after successful locking
611 CodeStub* slow_path = new MonitorEnterStub(object, lock, info);
612 __ load_stack_address_monitor(monitor_no, lock);
613 // for handling NullPointerException, use debug info representing just the lock stack before this monitorenter
614 __ lock_object(hdr, object, lock, scratch, slow_path, info_for_exception);
615 }
616
617
618 void LIRGenerator::monitor_exit(LIR_Opr object, LIR_Opr lock, LIR_Opr new_hdr, LIR_Opr scratch, int monitor_no) {
619 if (!GenerateSynchronizationCode) return;
620 // setup registers
621 LIR_Opr hdr = lock;
622 lock = new_hdr;
623 CodeStub* slow_path = new MonitorExitStub(lock, !UseHeavyMonitors, monitor_no);
624 __ load_stack_address_monitor(monitor_no, lock);
625 __ unlock_object(hdr, object, lock, scratch, slow_path);
626 }
627
628 #ifndef PRODUCT
629 void LIRGenerator::print_if_not_loaded(const NewInstance* new_instance) {
630 if (PrintNotLoaded && !new_instance->klass()->is_loaded()) {
631 tty->print_cr(" ###class not loaded at new bci %d", new_instance->printable_bci());
632 } else if (PrintNotLoaded && (!CompilerConfig::is_c1_only_no_jvmci() && new_instance->is_unresolved())) {
633 tty->print_cr(" ###class not resolved at new bci %d", new_instance->printable_bci());
634 }
635 }
636 #endif
637
638 void LIRGenerator::new_instance(LIR_Opr dst, ciInstanceKlass* klass, bool is_unresolved, LIR_Opr scratch1, LIR_Opr scratch2, LIR_Opr scratch3, LIR_Opr scratch4, LIR_Opr klass_reg, CodeEmitInfo* info) {
639 klass2reg_with_patching(klass_reg, klass, info, is_unresolved);
640 // If klass is not loaded we do not know if the klass has finalizers:
641 if (UseFastNewInstance && klass->is_loaded()
642 && !Klass::layout_helper_needs_slow_path(klass->layout_helper())) {
643
644 Runtime1::StubID stub_id = klass->is_initialized() ? Runtime1::fast_new_instance_id : Runtime1::fast_new_instance_init_check_id;
645
646 CodeStub* slow_path = new NewInstanceStub(klass_reg, dst, klass, info, stub_id);
647
648 assert(klass->is_loaded(), "must be loaded");
649 // allocate space for instance
650 assert(klass->size_helper() > 0, "illegal instance size");
651 const int instance_size = align_object_size(klass->size_helper());
652 __ allocate_object(dst, scratch1, scratch2, scratch3, scratch4,
653 oopDesc::header_size(), instance_size, klass_reg, !klass->is_initialized(), slow_path);
654 } else {
655 CodeStub* slow_path = new NewInstanceStub(klass_reg, dst, klass, info, Runtime1::new_instance_id);
656 __ branch(lir_cond_always, slow_path);
657 __ branch_destination(slow_path->continuation());
658 }
659 }
660
661
662 static bool is_constant_zero(Instruction* inst) {
663 IntConstant* c = inst->type()->as_IntConstant();
664 if (c) {
665 return (c->value() == 0);
666 }
667 return false;
668 }
669
670
671 static bool positive_constant(Instruction* inst) {
672 IntConstant* c = inst->type()->as_IntConstant();
673 if (c) {
674 return (c->value() >= 0);
675 }
676 return false;
736 if (src_type != NULL) {
737 if (src_type->element_type()->is_subtype_of(dst_type->element_type())) {
738 is_exact = true;
739 expected_type = dst_type;
740 }
741 }
742 }
743 // at least pass along a good guess
744 if (expected_type == NULL) expected_type = dst_exact_type;
745 if (expected_type == NULL) expected_type = src_declared_type;
746 if (expected_type == NULL) expected_type = dst_declared_type;
747
748 src_objarray = (src_exact_type && src_exact_type->is_obj_array_klass()) || (src_declared_type && src_declared_type->is_obj_array_klass());
749 dst_objarray = (dst_exact_type && dst_exact_type->is_obj_array_klass()) || (dst_declared_type && dst_declared_type->is_obj_array_klass());
750 }
751
752 // if a probable array type has been identified, figure out if any
753 // of the required checks for a fast case can be elided.
754 int flags = LIR_OpArrayCopy::all_flags;
755
756 if (!src_objarray)
757 flags &= ~LIR_OpArrayCopy::src_objarray;
758 if (!dst_objarray)
759 flags &= ~LIR_OpArrayCopy::dst_objarray;
760
761 if (!x->arg_needs_null_check(0))
762 flags &= ~LIR_OpArrayCopy::src_null_check;
763 if (!x->arg_needs_null_check(2))
764 flags &= ~LIR_OpArrayCopy::dst_null_check;
765
766
767 if (expected_type != NULL) {
768 Value length_limit = NULL;
769
770 IfOp* ifop = length->as_IfOp();
771 if (ifop != NULL) {
772 // look for expressions like min(v, a.length) which ends up as
773 // x > y ? y : x or x >= y ? y : x
774 if ((ifop->cond() == If::gtr || ifop->cond() == If::geq) &&
775 ifop->x() == ifop->fval() &&
1519 case T_FLOAT:
1520 if (c->as_jint_bits() != other->as_jint_bits()) continue;
1521 break;
1522 case T_LONG:
1523 case T_DOUBLE:
1524 if (c->as_jint_hi_bits() != other->as_jint_hi_bits()) continue;
1525 if (c->as_jint_lo_bits() != other->as_jint_lo_bits()) continue;
1526 break;
1527 case T_OBJECT:
1528 if (c->as_jobject() != other->as_jobject()) continue;
1529 break;
1530 default:
1531 break;
1532 }
1533 return _reg_for_constants.at(i);
1534 }
1535 }
1536
1537 LIR_Opr result = new_register(t);
1538 __ move((LIR_Opr)c, result);
1539 _constants.append(c);
1540 _reg_for_constants.append(result);
1541 return result;
1542 }
1543
1544 //------------------------field access--------------------------------------
1545
1546 void LIRGenerator::do_CompareAndSwap(Intrinsic* x, ValueType* type) {
1547 assert(x->number_of_arguments() == 4, "wrong type");
1548 LIRItem obj (x->argument_at(0), this); // object
1549 LIRItem offset(x->argument_at(1), this); // offset of field
1550 LIRItem cmp (x->argument_at(2), this); // value to compare with field
1551 LIRItem val (x->argument_at(3), this); // replace field with val if matches cmp
1552 assert(obj.type()->tag() == objectTag, "invalid type");
1553 assert(cmp.type()->tag() == type->tag(), "invalid type");
1554 assert(val.type()->tag() == type->tag(), "invalid type");
1555
1556 LIR_Opr result = access_atomic_cmpxchg_at(IN_HEAP, as_BasicType(type),
1557 obj, offset, cmp, val);
1558 set_result(x, result);
1559 }
1560
1561 // Comment copied form templateTable_i486.cpp
1562 // ----------------------------------------------------------------------------
1563 // Volatile variables demand their effects be made known to all CPU's in
1615 // load item if field not constant
1616 // because of code patching we cannot inline constants
1617 if (field_type == T_BYTE || field_type == T_BOOLEAN) {
1618 value.load_byte_item();
1619 } else {
1620 value.load_item();
1621 }
1622 } else {
1623 value.load_for_store(field_type);
1624 }
1625
1626 set_no_result(x);
1627
1628 #ifndef PRODUCT
1629 if (PrintNotLoaded && needs_patching) {
1630 tty->print_cr(" ###class not loaded at store_%s bci %d",
1631 x->is_static() ? "static" : "field", x->printable_bci());
1632 }
1633 #endif
1634
1635 if (x->needs_null_check() &&
1636 (needs_patching ||
1637 MacroAssembler::needs_explicit_null_check(x->offset()))) {
1638 // Emit an explicit null check because the offset is too large.
1639 // If the class is not loaded and the object is NULL, we need to deoptimize to throw a
1640 // NoClassDefFoundError in the interpreter instead of an implicit NPE from compiled code.
1641 __ null_check(object.result(), new CodeEmitInfo(info), /* deoptimize */ needs_patching);
1642 }
1643
1644 DecoratorSet decorators = IN_HEAP;
1645 if (is_volatile) {
1646 decorators |= MO_SEQ_CST;
1647 }
1648 if (needs_patching) {
1649 decorators |= C1_NEEDS_PATCHING;
1650 }
1651
1652 access_store_at(decorators, field_type, object, LIR_OprFact::intConst(x->offset()),
1653 value.result(), info != NULL ? new CodeEmitInfo(info) : NULL, info);
1654 }
1655
1656 void LIRGenerator::do_StoreIndexed(StoreIndexed* x) {
1657 assert(x->is_pinned(),"");
1658 bool needs_range_check = x->compute_needs_range_check();
1659 bool use_length = x->length() != NULL;
1660 bool obj_store = is_reference_type(x->elt_type());
1661 bool needs_store_check = obj_store && (x->value()->as_Constant() == NULL ||
1662 !get_jobject_constant(x->value())->is_null_object() ||
1663 x->should_profile());
1664
1665 LIRItem array(x->array(), this);
1666 LIRItem index(x->index(), this);
1667 LIRItem value(x->value(), this);
1668 LIRItem length(this);
1669
1670 array.load_item();
1671 index.load_nonconstant();
1672
1673 if (use_length && needs_range_check) {
1674 length.set_instruction(x->length());
1675 length.load_item();
1676
1677 }
1678 if (needs_store_check || x->check_boolean()) {
1679 value.load_item();
1680 } else {
1681 value.load_for_store(x->elt_type());
1682 }
1683
1684 set_no_result(x);
1685
1686 // the CodeEmitInfo must be duplicated for each different
1687 // LIR-instruction because spilling can occur anywhere between two
1688 // instructions and so the debug information must be different
1689 CodeEmitInfo* range_check_info = state_for(x);
1690 CodeEmitInfo* null_check_info = NULL;
1691 if (x->needs_null_check()) {
1692 null_check_info = new CodeEmitInfo(range_check_info);
1693 }
1694
1695 if (GenerateRangeChecks && needs_range_check) {
1696 if (use_length) {
1697 __ cmp(lir_cond_belowEqual, length.result(), index.result());
1698 __ branch(lir_cond_belowEqual, new RangeCheckStub(range_check_info, index.result(), array.result()));
1699 } else {
1700 array_range_check(array.result(), index.result(), null_check_info, range_check_info);
1701 // range_check also does the null check
1702 null_check_info = NULL;
1703 }
1704 }
1705
1706 if (GenerateArrayStoreCheck && needs_store_check) {
1707 CodeEmitInfo* store_check_info = new CodeEmitInfo(range_check_info);
1708 array_store_check(value.result(), array.result(), store_check_info, x->profiled_method(), x->profiled_bci());
1709 }
1710
1711 DecoratorSet decorators = IN_HEAP | IS_ARRAY;
1712 if (x->check_boolean()) {
1713 decorators |= C1_MASK_BOOLEAN;
1714 }
1715
1716 access_store_at(decorators, x->elt_type(), array, index.result(), value.result(),
1717 NULL, null_check_info);
1718 }
1719
1720 void LIRGenerator::access_load_at(DecoratorSet decorators, BasicType type,
1721 LIRItem& base, LIR_Opr offset, LIR_Opr result,
1722 CodeEmitInfo* patch_info, CodeEmitInfo* load_emit_info) {
1723 decorators |= ACCESS_READ;
1724 LIRAccess access(this, decorators, base, offset, type, patch_info, load_emit_info);
1725 if (access.is_raw()) {
1726 _barrier_set->BarrierSetC1::load_at(access, result);
1727 } else {
1728 _barrier_set->load_at(access, result);
1729 }
1730 }
1731
1732 void LIRGenerator::access_load(DecoratorSet decorators, BasicType type,
1733 LIR_Opr addr, LIR_Opr result) {
1734 decorators |= ACCESS_READ;
1735 LIRAccess access(this, decorators, LIR_OprFact::illegalOpr, LIR_OprFact::illegalOpr, type);
1736 access.set_resolved_addr(addr);
1737 if (access.is_raw()) {
1778 return _barrier_set->BarrierSetC1::atomic_xchg_at(access, value);
1779 } else {
1780 return _barrier_set->atomic_xchg_at(access, value);
1781 }
1782 }
1783
1784 LIR_Opr LIRGenerator::access_atomic_add_at(DecoratorSet decorators, BasicType type,
1785 LIRItem& base, LIRItem& offset, LIRItem& value) {
1786 decorators |= ACCESS_READ;
1787 decorators |= ACCESS_WRITE;
1788 // Atomic operations are SEQ_CST by default
1789 decorators |= ((decorators & MO_DECORATOR_MASK) == 0) ? MO_SEQ_CST : 0;
1790 LIRAccess access(this, decorators, base, offset, type);
1791 if (access.is_raw()) {
1792 return _barrier_set->BarrierSetC1::atomic_add_at(access, value);
1793 } else {
1794 return _barrier_set->atomic_add_at(access, value);
1795 }
1796 }
1797
1798 void LIRGenerator::do_LoadField(LoadField* x) {
1799 bool needs_patching = x->needs_patching();
1800 bool is_volatile = x->field()->is_volatile();
1801 BasicType field_type = x->field_type();
1802
1803 CodeEmitInfo* info = NULL;
1804 if (needs_patching) {
1805 assert(x->explicit_null_check() == NULL, "can't fold null check into patching field access");
1806 info = state_for(x, x->state_before());
1807 } else if (x->needs_null_check()) {
1808 NullCheck* nc = x->explicit_null_check();
1809 if (nc == NULL) {
1810 info = state_for(x);
1811 } else {
1812 info = state_for(nc);
1813 }
1814 }
1815
1816 LIRItem object(x->obj(), this);
1817
1818 object.load_item();
1819
1820 #ifndef PRODUCT
1821 if (PrintNotLoaded && needs_patching) {
1822 tty->print_cr(" ###class not loaded at load_%s bci %d",
1823 x->is_static() ? "static" : "field", x->printable_bci());
1824 }
1825 #endif
1826
1827 bool stress_deopt = StressLoopInvariantCodeMotion && info && info->deoptimize_on_exception();
1828 if (x->needs_null_check() &&
1829 (needs_patching ||
1830 MacroAssembler::needs_explicit_null_check(x->offset()) ||
1831 stress_deopt)) {
1832 LIR_Opr obj = object.result();
1833 if (stress_deopt) {
1834 obj = new_register(T_OBJECT);
1835 __ move(LIR_OprFact::oopConst(NULL), obj);
1836 }
1837 // Emit an explicit null check because the offset is too large.
1838 // If the class is not loaded and the object is NULL, we need to deoptimize to throw a
1839 // NoClassDefFoundError in the interpreter instead of an implicit NPE from compiled code.
1840 __ null_check(obj, new CodeEmitInfo(info), /* deoptimize */ needs_patching);
1841 }
1842
1843 DecoratorSet decorators = IN_HEAP;
1844 if (is_volatile) {
1845 decorators |= MO_SEQ_CST;
1846 }
1847 if (needs_patching) {
1848 decorators |= C1_NEEDS_PATCHING;
1849 }
1850
1851 LIR_Opr result = rlock_result(x, field_type);
1852 access_load_at(decorators, field_type,
1853 object, LIR_OprFact::intConst(x->offset()), result,
1854 info ? new CodeEmitInfo(info) : NULL, info);
1855 }
1856
1857 // int/long jdk.internal.util.Preconditions.checkIndex
1858 void LIRGenerator::do_PreconditionsCheckIndex(Intrinsic* x, BasicType type) {
1859 assert(x->number_of_arguments() == 3, "wrong type");
1860 LIRItem index(x->argument_at(0), this);
1861 LIRItem length(x->argument_at(1), this);
1862 LIRItem oobef(x->argument_at(2), this);
1863
1864 index.load_item();
1865 length.load_item();
1866 oobef.load_item();
1867
1868 LIR_Opr result = rlock_result(x);
1869 // x->state() is created from copy_state_for_exception, it does not contains arguments
1870 // we should prepare them before entering into interpreter mode due to deoptimization.
1871 ValueStack* state = x->state();
1872 for (int i = 0; i < x->number_of_arguments(); i++) {
1873 Value arg = x->argument_at(i);
1874 state->push(arg->type(), arg);
1979 __ move(LIR_OprFact::oopConst(NULL), obj);
1980 __ null_check(obj, new CodeEmitInfo(null_check_info));
1981 }
1982 }
1983
1984 if (GenerateRangeChecks && needs_range_check) {
1985 if (StressLoopInvariantCodeMotion && range_check_info->deoptimize_on_exception()) {
1986 __ branch(lir_cond_always, new RangeCheckStub(range_check_info, index.result(), array.result()));
1987 } else if (use_length) {
1988 // TODO: use a (modified) version of array_range_check that does not require a
1989 // constant length to be loaded to a register
1990 __ cmp(lir_cond_belowEqual, length.result(), index.result());
1991 __ branch(lir_cond_belowEqual, new RangeCheckStub(range_check_info, index.result(), array.result()));
1992 } else {
1993 array_range_check(array.result(), index.result(), null_check_info, range_check_info);
1994 // The range check performs the null check, so clear it out for the load
1995 null_check_info = NULL;
1996 }
1997 }
1998
1999 DecoratorSet decorators = IN_HEAP | IS_ARRAY;
2000
2001 LIR_Opr result = rlock_result(x, x->elt_type());
2002 access_load_at(decorators, x->elt_type(),
2003 array, index.result(), result,
2004 NULL, null_check_info);
2005 }
2006
2007
2008 void LIRGenerator::do_NullCheck(NullCheck* x) {
2009 if (x->can_trap()) {
2010 LIRItem value(x->obj(), this);
2011 value.load_item();
2012 CodeEmitInfo* info = state_for(x);
2013 __ null_check(value.result(), info);
2014 }
2015 }
2016
2017
2018 void LIRGenerator::do_TypeCast(TypeCast* x) {
2019 LIRItem value(x->obj(), this);
2020 value.load_item();
2021 // the result is the same as from the node we are casting
2022 set_result(x, value.result());
2023 }
2024
2025
2026 void LIRGenerator::do_Throw(Throw* x) {
2485 Compilation* comp = Compilation::current();
2486 if (do_update) {
2487 // try to find exact type, using CHA if possible, so that loading
2488 // the klass from the object can be avoided
2489 ciType* type = obj->exact_type();
2490 if (type == NULL) {
2491 type = obj->declared_type();
2492 type = comp->cha_exact_type(type);
2493 }
2494 assert(type == NULL || type->is_klass(), "type should be class");
2495 exact_klass = (type != NULL && type->is_loaded()) ? (ciKlass*)type : NULL;
2496
2497 do_update = exact_klass == NULL || ciTypeEntries::valid_ciklass(profiled_k) != exact_klass;
2498 }
2499
2500 if (!do_null && !do_update) {
2501 return result;
2502 }
2503
2504 ciKlass* exact_signature_k = NULL;
2505 if (do_update) {
2506 // Is the type from the signature exact (the only one possible)?
2507 exact_signature_k = signature_at_call_k->exact_klass();
2508 if (exact_signature_k == NULL) {
2509 exact_signature_k = comp->cha_exact_type(signature_at_call_k);
2510 } else {
2511 result = exact_signature_k;
2512 // Known statically. No need to emit any code: prevent
2513 // LIR_Assembler::emit_profile_type() from emitting useless code
2514 profiled_k = ciTypeEntries::with_status(result, profiled_k);
2515 }
2516 // exact_klass and exact_signature_k can be both non NULL but
2517 // different if exact_klass is loaded after the ciObject for
2518 // exact_signature_k is created.
2519 if (exact_klass == NULL && exact_signature_k != NULL && exact_klass != exact_signature_k) {
2520 // sometimes the type of the signature is better than the best type
2521 // the compiler has
2522 exact_klass = exact_signature_k;
2523 }
2524 if (callee_signature_k != NULL &&
2525 callee_signature_k != signature_at_call_k) {
2570 assert(!src->is_illegal(), "check");
2571 BasicType t = src->type();
2572 if (is_reference_type(t)) {
2573 intptr_t profiled_k = parameters->type(j);
2574 Local* local = x->state()->local_at(java_index)->as_Local();
2575 ciKlass* exact = profile_type(md, md->byte_offset_of_slot(parameters_type_data, ParametersTypeData::type_offset(0)),
2576 in_bytes(ParametersTypeData::type_offset(j)) - in_bytes(ParametersTypeData::type_offset(0)),
2577 profiled_k, local, mdp, false, local->declared_type()->as_klass(), NULL);
2578 // If the profile is known statically set it once for all and do not emit any code
2579 if (exact != NULL) {
2580 md->set_parameter_type(j, exact);
2581 }
2582 j++;
2583 }
2584 java_index += type2size[t];
2585 }
2586 }
2587 }
2588 }
2589
2590 void LIRGenerator::do_Base(Base* x) {
2591 __ std_entry(LIR_OprFact::illegalOpr);
2592 // Emit moves from physical registers / stack slots to virtual registers
2593 CallingConvention* args = compilation()->frame_map()->incoming_arguments();
2594 IRScope* irScope = compilation()->hir()->top_scope();
2595 int java_index = 0;
2596 for (int i = 0; i < args->length(); i++) {
2597 LIR_Opr src = args->at(i);
2598 assert(!src->is_illegal(), "check");
2599 BasicType t = src->type();
2600
2601 // Types which are smaller than int are passed as int, so
2602 // correct the type which passed.
2603 switch (t) {
2604 case T_BYTE:
2605 case T_BOOLEAN:
2606 case T_SHORT:
2607 case T_CHAR:
2608 t = T_INT;
2609 break;
2654 LIR_Opr lock = syncLockOpr();
2655 __ load_stack_address_monitor(0, lock);
2656
2657 CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, SynchronizationEntryBCI), NULL, x->check_flag(Instruction::DeoptimizeOnException));
2658 CodeStub* slow_path = new MonitorEnterStub(obj, lock, info);
2659
2660 // receiver is guaranteed non-NULL so don't need CodeEmitInfo
2661 __ lock_object(syncTempOpr(), obj, lock, new_register(T_OBJECT), slow_path, NULL);
2662 }
2663 }
2664 if (compilation()->age_code()) {
2665 CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, 0), NULL, false);
2666 decrement_age(info);
2667 }
2668 // increment invocation counters if needed
2669 if (!method()->is_accessor()) { // Accessors do not have MDOs, so no counting.
2670 profile_parameters(x);
2671 CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, SynchronizationEntryBCI), NULL, false);
2672 increment_invocation_counter(info);
2673 }
2674
2675 // all blocks with a successor must end with an unconditional jump
2676 // to the successor even if they are consecutive
2677 __ jump(x->default_sux());
2678 }
2679
2680
2681 void LIRGenerator::do_OsrEntry(OsrEntry* x) {
2682 // construct our frame and model the production of incoming pointer
2683 // to the OSR buffer.
2684 __ osr_entry(LIR_Assembler::osrBufferPointer());
2685 LIR_Opr result = rlock_result(x);
2686 __ move(LIR_Assembler::osrBufferPointer(), result);
2687 }
2688
2689
2690 void LIRGenerator::invoke_load_arguments(Invoke* x, LIRItemList* args, const LIR_OprList* arg_list) {
2691 assert(args->length() == arg_list->length(),
2692 "args=%d, arg_list=%d", args->length(), arg_list->length());
2693 for (int i = x->has_receiver() ? 1 : 0; i < args->length(); i++) {
2694 LIRItem* param = args->at(i);
2695 LIR_Opr loc = arg_list->at(i);
2696 if (loc->is_register()) {
2697 param->load_item_force(loc);
2698 } else {
2699 LIR_Address* addr = loc->as_address_ptr();
2700 param->load_for_store(addr->type());
2701 if (addr->type() == T_OBJECT) {
2702 __ move_wide(param->result(), addr);
2703 } else
2704 __ move(param->result(), addr);
2705 }
2706 }
2707
2708 if (x->has_receiver()) {
2709 LIRItem* receiver = args->at(0);
2710 LIR_Opr loc = arg_list->at(0);
2711 if (loc->is_register()) {
2712 receiver->load_item_force(loc);
2713 } else {
2714 assert(loc->is_address(), "just checking");
2715 receiver->load_for_store(T_OBJECT);
2716 __ move_wide(receiver->result(), loc->as_address_ptr());
2717 }
2718 }
2719 }
2720
2721
2722 // Visits all arguments, returns appropriate items without loading them
2723 LIRItemList* LIRGenerator::invoke_visit_arguments(Invoke* x) {
2724 LIRItemList* argument_items = new LIRItemList();
2725 if (x->has_receiver()) {
2851 __ move(tmp, reg);
2852 }
2853
2854
2855
2856 // Code for : x->x() {x->cond()} x->y() ? x->tval() : x->fval()
2857 void LIRGenerator::do_IfOp(IfOp* x) {
2858 #ifdef ASSERT
2859 {
2860 ValueTag xtag = x->x()->type()->tag();
2861 ValueTag ttag = x->tval()->type()->tag();
2862 assert(xtag == intTag || xtag == objectTag, "cannot handle others");
2863 assert(ttag == addressTag || ttag == intTag || ttag == objectTag || ttag == longTag, "cannot handle others");
2864 assert(ttag == x->fval()->type()->tag(), "cannot handle others");
2865 }
2866 #endif
2867
2868 LIRItem left(x->x(), this);
2869 LIRItem right(x->y(), this);
2870 left.load_item();
2871 if (can_inline_as_constant(right.value())) {
2872 right.dont_load_item();
2873 } else {
2874 right.load_item();
2875 }
2876
2877 LIRItem t_val(x->tval(), this);
2878 LIRItem f_val(x->fval(), this);
2879 t_val.dont_load_item();
2880 f_val.dont_load_item();
2881 LIR_Opr reg = rlock_result(x);
2882
2883 __ cmp(lir_cond(x->cond()), left.result(), right.result());
2884 __ cmove(lir_cond(x->cond()), t_val.result(), f_val.result(), reg, as_BasicType(x->x()->type()));
2885 }
2886
2887 #ifdef JFR_HAVE_INTRINSICS
2888
2889 void LIRGenerator::do_getEventWriter(Intrinsic* x) {
2890 LabelObj* L_end = new LabelObj();
2891
2892 // FIXME T_ADDRESS should actually be T_METADATA but it can't because the
2893 // meaning of these two is mixed up (see JDK-8026837).
2894 LIR_Address* jobj_addr = new LIR_Address(getThreadPointer(),
2895 in_bytes(THREAD_LOCAL_WRITER_OFFSET_JFR),
2896 T_ADDRESS);
2897 LIR_Opr result = rlock_result(x);
2898 __ move(LIR_OprFact::oopConst(NULL), result);
2899 LIR_Opr jobj = new_register(T_METADATA);
2900 __ move_wide(jobj_addr, jobj);
2901 __ cmp(lir_cond_equal, jobj, LIR_OprFact::metadataConst(0));
2902 __ branch(lir_cond_equal, L_end->label());
2903
2904 access_load(IN_NATIVE, T_OBJECT, LIR_OprFact::address(new LIR_Address(jobj, T_OBJECT)), result);
3160 if (x->recv() != NULL || x->nb_profiled_args() > 0) {
3161 profile_parameters_at_call(x);
3162 }
3163
3164 if (x->recv() != NULL) {
3165 LIRItem value(x->recv(), this);
3166 value.load_item();
3167 recv = new_register(T_OBJECT);
3168 __ move(value.result(), recv);
3169 }
3170 __ profile_call(x->method(), x->bci_of_invoke(), x->callee(), mdo, recv, tmp, x->known_holder());
3171 }
3172
3173 void LIRGenerator::do_ProfileReturnType(ProfileReturnType* x) {
3174 int bci = x->bci_of_invoke();
3175 ciMethodData* md = x->method()->method_data_or_null();
3176 assert(md != NULL, "Sanity");
3177 ciProfileData* data = md->bci_to_data(bci);
3178 if (data != NULL) {
3179 assert(data->is_CallTypeData() || data->is_VirtualCallTypeData(), "wrong profile data type");
3180 ciReturnTypeEntry* ret = data->is_CallTypeData() ? ((ciCallTypeData*)data)->ret() : ((ciVirtualCallTypeData*)data)->ret();
3181 LIR_Opr mdp = LIR_OprFact::illegalOpr;
3182
3183 bool ignored_will_link;
3184 ciSignature* signature_at_call = NULL;
3185 x->method()->get_method_at_bci(bci, ignored_will_link, &signature_at_call);
3186
3187 // The offset within the MDO of the entry to update may be too large
3188 // to be used in load/store instructions on some platforms. So have
3189 // profile_type() compute the address of the profile in a register.
3190 ciKlass* exact = profile_type(md, md->byte_offset_of_slot(data, ret->type_offset()), 0,
3191 ret->type(), x->ret(), mdp,
3192 !x->needs_null_check(),
3193 signature_at_call->return_type()->as_klass(),
3194 x->callee()->signature()->return_type()->as_klass());
3195 if (exact != NULL) {
3196 md->set_return_type(bci, exact);
3197 }
3198 }
3199 }
3200
3201 void LIRGenerator::do_ProfileInvoke(ProfileInvoke* x) {
3202 // We can safely ignore accessors here, since c2 will inline them anyway,
3203 // accessors are also always mature.
3204 if (!x->inlinee()->is_accessor()) {
3205 CodeEmitInfo* info = state_for(x, x->state(), true);
3206 // Notify the runtime very infrequently only to take care of counter overflows
3207 int freq_log = Tier23InlineeNotifyFreqLog;
3208 double scale;
3209 if (_method->has_option_value(CompileCommand::CompileThresholdScaling, scale)) {
3210 freq_log = CompilerConfig::scaled_freq_log(freq_log, scale);
3211 }
3212 increment_event_counter_impl(info, x->inlinee(), LIR_OprFact::intConst(InvocationCounter::count_increment), right_n_bits(freq_log), InvocationEntryBci, false, true);
3213 }
3214 }
3215
3216 void LIRGenerator::increment_backedge_counter_conditionally(LIR_Condition cond, LIR_Opr left, LIR_Opr right, CodeEmitInfo* info, int left_bci, int right_bci, int bci) {
3217 if (compilation()->is_profiling()) {
3218 #if defined(X86) && !defined(_LP64)
3219 // BEWARE! On 32-bit x86 cmp clobbers its left argument so we need a temp copy.
3220 LIR_Opr left_copy = new_register(left->type());
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "c1/c1_Compilation.hpp"
27 #include "c1/c1_Defs.hpp"
28 #include "c1/c1_FrameMap.hpp"
29 #include "c1/c1_Instruction.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_LIRGenerator.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciFlatArrayKlass.hpp"
35 #include "ci/ciInlineKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "ci/ciObjArray.hpp"
38 #include "ci/ciUtilities.hpp"
39 #include "gc/shared/barrierSet.hpp"
40 #include "gc/shared/c1/barrierSetC1.hpp"
41 #include "oops/klass.inline.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "runtime/stubRoutines.hpp"
44 #include "runtime/vm_version.hpp"
45 #include "utilities/bitMap.inline.hpp"
46 #include "utilities/macros.hpp"
47 #include "utilities/powerOfTwo.hpp"
48
49 #ifdef ASSERT
50 #define __ gen()->lir(__FILE__, __LINE__)->
51 #else
52 #define __ gen()->lir()->
53 #endif
54
55 #ifndef PATCHED_ADDR
195 ResolveNode* source = source_node(src);
196 source->append(destination_node(dest));
197 }
198
199
200 //--------------------------------------------------------------
201 // LIRItem
202
203 void LIRItem::set_result(LIR_Opr opr) {
204 assert(value()->operand()->is_illegal() || value()->operand()->is_constant(), "operand should never change");
205 value()->set_operand(opr);
206
207 if (opr->is_virtual()) {
208 _gen->_instruction_for_operand.at_put_grow(opr->vreg_number(), value(), NULL);
209 }
210
211 _result = opr;
212 }
213
214 void LIRItem::load_item() {
215 assert(!_gen->in_conditional_code(), "LIRItem cannot be loaded in conditional code");
216
217 if (result()->is_illegal()) {
218 // update the items result
219 _result = value()->operand();
220 }
221 if (!result()->is_register()) {
222 LIR_Opr reg = _gen->new_register(value()->type());
223 __ move(result(), reg);
224 if (result()->is_constant()) {
225 _result = reg;
226 } else {
227 set_result(reg);
228 }
229 }
230 }
231
232
233 void LIRItem::load_for_store(BasicType type) {
234 if (_gen->can_store_as_constant(value(), type)) {
235 _result = value()->operand();
236 if (!_result->is_constant()) {
592 assert(right_op != result_op, "malformed");
593 __ move(left_op, result_op);
594 left_op = result_op;
595 }
596
597 switch(code) {
598 case Bytecodes::_iand:
599 case Bytecodes::_land: __ logical_and(left_op, right_op, result_op); break;
600
601 case Bytecodes::_ior:
602 case Bytecodes::_lor: __ logical_or(left_op, right_op, result_op); break;
603
604 case Bytecodes::_ixor:
605 case Bytecodes::_lxor: __ logical_xor(left_op, right_op, result_op); break;
606
607 default: ShouldNotReachHere();
608 }
609 }
610
611
612 void LIRGenerator::monitor_enter(LIR_Opr object, LIR_Opr lock, LIR_Opr hdr, LIR_Opr scratch, int monitor_no,
613 CodeEmitInfo* info_for_exception, CodeEmitInfo* info, CodeStub* throw_imse_stub) {
614 if (!GenerateSynchronizationCode) return;
615 // for slow path, use debug info for state after successful locking
616 CodeStub* slow_path = new MonitorEnterStub(object, lock, info, throw_imse_stub, scratch);
617 __ load_stack_address_monitor(monitor_no, lock);
618 // for handling NullPointerException, use debug info representing just the lock stack before this monitorenter
619 __ lock_object(hdr, object, lock, scratch, slow_path, info_for_exception, throw_imse_stub);
620 }
621
622
623 void LIRGenerator::monitor_exit(LIR_Opr object, LIR_Opr lock, LIR_Opr new_hdr, LIR_Opr scratch, int monitor_no) {
624 if (!GenerateSynchronizationCode) return;
625 // setup registers
626 LIR_Opr hdr = lock;
627 lock = new_hdr;
628 CodeStub* slow_path = new MonitorExitStub(lock, !UseHeavyMonitors, monitor_no);
629 __ load_stack_address_monitor(monitor_no, lock);
630 __ unlock_object(hdr, object, lock, scratch, slow_path);
631 }
632
633 #ifndef PRODUCT
634 void LIRGenerator::print_if_not_loaded(const NewInstance* new_instance) {
635 if (PrintNotLoaded && !new_instance->klass()->is_loaded()) {
636 tty->print_cr(" ###class not loaded at new bci %d", new_instance->printable_bci());
637 } else if (PrintNotLoaded && (!CompilerConfig::is_c1_only_no_jvmci() && new_instance->is_unresolved())) {
638 tty->print_cr(" ###class not resolved at new bci %d", new_instance->printable_bci());
639 }
640 }
641 #endif
642
643 void LIRGenerator::new_instance(LIR_Opr dst, ciInstanceKlass* klass, bool is_unresolved, bool allow_inline, LIR_Opr scratch1, LIR_Opr scratch2, LIR_Opr scratch3, LIR_Opr scratch4, LIR_Opr klass_reg, CodeEmitInfo* info) {
644 if (allow_inline) {
645 assert(!is_unresolved && klass->is_loaded(), "inline type klass should be resolved");
646 __ metadata2reg(klass->constant_encoding(), klass_reg);
647 } else {
648 klass2reg_with_patching(klass_reg, klass, info, is_unresolved);
649 }
650 // If klass is not loaded we do not know if the klass has finalizers or is an unexpected inline klass
651 if (UseFastNewInstance && klass->is_loaded() && (allow_inline || !klass->is_inlinetype())
652 && !Klass::layout_helper_needs_slow_path(klass->layout_helper())) {
653
654 Runtime1::StubID stub_id = klass->is_initialized() ? Runtime1::fast_new_instance_id : Runtime1::fast_new_instance_init_check_id;
655
656 CodeStub* slow_path = new NewInstanceStub(klass_reg, dst, klass, info, stub_id);
657
658 assert(klass->is_loaded(), "must be loaded");
659 // allocate space for instance
660 assert(klass->size_helper() > 0, "illegal instance size");
661 const int instance_size = align_object_size(klass->size_helper());
662 __ allocate_object(dst, scratch1, scratch2, scratch3, scratch4,
663 oopDesc::header_size(), instance_size, klass_reg, !klass->is_initialized(), slow_path);
664 } else {
665 CodeStub* slow_path = new NewInstanceStub(klass_reg, dst, klass, info, allow_inline ? Runtime1::new_instance_id : Runtime1::new_instance_no_inline_id);
666 __ jump(slow_path);
667 __ branch_destination(slow_path->continuation());
668 }
669 }
670
671
672 static bool is_constant_zero(Instruction* inst) {
673 IntConstant* c = inst->type()->as_IntConstant();
674 if (c) {
675 return (c->value() == 0);
676 }
677 return false;
678 }
679
680
681 static bool positive_constant(Instruction* inst) {
682 IntConstant* c = inst->type()->as_IntConstant();
683 if (c) {
684 return (c->value() >= 0);
685 }
686 return false;
746 if (src_type != NULL) {
747 if (src_type->element_type()->is_subtype_of(dst_type->element_type())) {
748 is_exact = true;
749 expected_type = dst_type;
750 }
751 }
752 }
753 // at least pass along a good guess
754 if (expected_type == NULL) expected_type = dst_exact_type;
755 if (expected_type == NULL) expected_type = src_declared_type;
756 if (expected_type == NULL) expected_type = dst_declared_type;
757
758 src_objarray = (src_exact_type && src_exact_type->is_obj_array_klass()) || (src_declared_type && src_declared_type->is_obj_array_klass());
759 dst_objarray = (dst_exact_type && dst_exact_type->is_obj_array_klass()) || (dst_declared_type && dst_declared_type->is_obj_array_klass());
760 }
761
762 // if a probable array type has been identified, figure out if any
763 // of the required checks for a fast case can be elided.
764 int flags = LIR_OpArrayCopy::all_flags;
765
766 if (!src->is_loaded_flattened_array() && !dst->is_loaded_flattened_array()) {
767 flags &= ~LIR_OpArrayCopy::always_slow_path;
768 }
769 if (!src->maybe_flattened_array()) {
770 flags &= ~LIR_OpArrayCopy::src_inlinetype_check;
771 }
772 if (!dst->maybe_flattened_array() && !dst->maybe_null_free_array()) {
773 flags &= ~LIR_OpArrayCopy::dst_inlinetype_check;
774 }
775
776 if (!src_objarray)
777 flags &= ~LIR_OpArrayCopy::src_objarray;
778 if (!dst_objarray)
779 flags &= ~LIR_OpArrayCopy::dst_objarray;
780
781 if (!x->arg_needs_null_check(0))
782 flags &= ~LIR_OpArrayCopy::src_null_check;
783 if (!x->arg_needs_null_check(2))
784 flags &= ~LIR_OpArrayCopy::dst_null_check;
785
786
787 if (expected_type != NULL) {
788 Value length_limit = NULL;
789
790 IfOp* ifop = length->as_IfOp();
791 if (ifop != NULL) {
792 // look for expressions like min(v, a.length) which ends up as
793 // x > y ? y : x or x >= y ? y : x
794 if ((ifop->cond() == If::gtr || ifop->cond() == If::geq) &&
795 ifop->x() == ifop->fval() &&
1539 case T_FLOAT:
1540 if (c->as_jint_bits() != other->as_jint_bits()) continue;
1541 break;
1542 case T_LONG:
1543 case T_DOUBLE:
1544 if (c->as_jint_hi_bits() != other->as_jint_hi_bits()) continue;
1545 if (c->as_jint_lo_bits() != other->as_jint_lo_bits()) continue;
1546 break;
1547 case T_OBJECT:
1548 if (c->as_jobject() != other->as_jobject()) continue;
1549 break;
1550 default:
1551 break;
1552 }
1553 return _reg_for_constants.at(i);
1554 }
1555 }
1556
1557 LIR_Opr result = new_register(t);
1558 __ move((LIR_Opr)c, result);
1559 if (!in_conditional_code()) {
1560 _constants.append(c);
1561 _reg_for_constants.append(result);
1562 }
1563 return result;
1564 }
1565
1566 void LIRGenerator::set_in_conditional_code(bool v) {
1567 assert(v != _in_conditional_code, "must change state");
1568 _in_conditional_code = v;
1569 }
1570
1571
1572 //------------------------field access--------------------------------------
1573
1574 void LIRGenerator::do_CompareAndSwap(Intrinsic* x, ValueType* type) {
1575 assert(x->number_of_arguments() == 4, "wrong type");
1576 LIRItem obj (x->argument_at(0), this); // object
1577 LIRItem offset(x->argument_at(1), this); // offset of field
1578 LIRItem cmp (x->argument_at(2), this); // value to compare with field
1579 LIRItem val (x->argument_at(3), this); // replace field with val if matches cmp
1580 assert(obj.type()->tag() == objectTag, "invalid type");
1581 assert(cmp.type()->tag() == type->tag(), "invalid type");
1582 assert(val.type()->tag() == type->tag(), "invalid type");
1583
1584 LIR_Opr result = access_atomic_cmpxchg_at(IN_HEAP, as_BasicType(type),
1585 obj, offset, cmp, val);
1586 set_result(x, result);
1587 }
1588
1589 // Comment copied form templateTable_i486.cpp
1590 // ----------------------------------------------------------------------------
1591 // Volatile variables demand their effects be made known to all CPU's in
1643 // load item if field not constant
1644 // because of code patching we cannot inline constants
1645 if (field_type == T_BYTE || field_type == T_BOOLEAN) {
1646 value.load_byte_item();
1647 } else {
1648 value.load_item();
1649 }
1650 } else {
1651 value.load_for_store(field_type);
1652 }
1653
1654 set_no_result(x);
1655
1656 #ifndef PRODUCT
1657 if (PrintNotLoaded && needs_patching) {
1658 tty->print_cr(" ###class not loaded at store_%s bci %d",
1659 x->is_static() ? "static" : "field", x->printable_bci());
1660 }
1661 #endif
1662
1663 if (!inline_type_field_access_prolog(x)) {
1664 // Field store will always deopt due to unloaded field or holder klass
1665 return;
1666 }
1667
1668 if (x->needs_null_check() &&
1669 (needs_patching ||
1670 MacroAssembler::needs_explicit_null_check(x->offset()))) {
1671 // Emit an explicit null check because the offset is too large.
1672 // If the class is not loaded and the object is NULL, we need to deoptimize to throw a
1673 // NoClassDefFoundError in the interpreter instead of an implicit NPE from compiled code.
1674 __ null_check(object.result(), new CodeEmitInfo(info), /* deoptimize */ needs_patching);
1675 }
1676
1677 DecoratorSet decorators = IN_HEAP;
1678 if (is_volatile) {
1679 decorators |= MO_SEQ_CST;
1680 }
1681 if (needs_patching) {
1682 decorators |= C1_NEEDS_PATCHING;
1683 }
1684
1685 access_store_at(decorators, field_type, object, LIR_OprFact::intConst(x->offset()),
1686 value.result(), info != NULL ? new CodeEmitInfo(info) : NULL, info);
1687 }
1688
1689 // FIXME -- I can't find any other way to pass an address to access_load_at().
1690 class TempResolvedAddress: public Instruction {
1691 public:
1692 TempResolvedAddress(ValueType* type, LIR_Opr addr) : Instruction(type) {
1693 set_operand(addr);
1694 }
1695 virtual void input_values_do(ValueVisitor*) {}
1696 virtual void visit(InstructionVisitor* v) {}
1697 virtual const char* name() const { return "TempResolvedAddress"; }
1698 };
1699
1700 LIR_Opr LIRGenerator::get_and_load_element_address(LIRItem& array, LIRItem& index) {
1701 ciType* array_type = array.value()->declared_type();
1702 ciFlatArrayKlass* flat_array_klass = array_type->as_flat_array_klass();
1703 assert(flat_array_klass->is_loaded(), "must be");
1704
1705 int array_header_size = flat_array_klass->array_header_in_bytes();
1706 int shift = flat_array_klass->log2_element_size();
1707
1708 #ifndef _LP64
1709 LIR_Opr index_op = new_register(T_INT);
1710 // FIXME -- on 32-bit, the shift below can overflow, so we need to check that
1711 // the top (shift+1) bits of index_op must be zero, or
1712 // else throw ArrayIndexOutOfBoundsException
1713 if (index.result()->is_constant()) {
1714 jint const_index = index.result()->as_jint();
1715 __ move(LIR_OprFact::intConst(const_index << shift), index_op);
1716 } else {
1717 __ shift_left(index_op, shift, index.result());
1718 }
1719 #else
1720 LIR_Opr index_op = new_register(T_LONG);
1721 if (index.result()->is_constant()) {
1722 jint const_index = index.result()->as_jint();
1723 __ move(LIR_OprFact::longConst(const_index << shift), index_op);
1724 } else {
1725 __ convert(Bytecodes::_i2l, index.result(), index_op);
1726 // Need to shift manually, as LIR_Address can scale only up to 3.
1727 __ shift_left(index_op, shift, index_op);
1728 }
1729 #endif
1730
1731 LIR_Opr elm_op = new_pointer_register();
1732 LIR_Address* elm_address = generate_address(array.result(), index_op, 0, array_header_size, T_ADDRESS);
1733 __ leal(LIR_OprFact::address(elm_address), elm_op);
1734 return elm_op;
1735 }
1736
1737 void LIRGenerator::access_sub_element(LIRItem& array, LIRItem& index, LIR_Opr& result, ciField* field, int sub_offset) {
1738 assert(field != NULL, "Need a subelement type specified");
1739
1740 // Find the starting address of the source (inside the array)
1741 LIR_Opr elm_op = get_and_load_element_address(array, index);
1742
1743 BasicType subelt_type = field->type()->basic_type();
1744 TempResolvedAddress* elm_resolved_addr = new TempResolvedAddress(as_ValueType(subelt_type), elm_op);
1745 LIRItem elm_item(elm_resolved_addr, this);
1746
1747 DecoratorSet decorators = IN_HEAP;
1748 access_load_at(decorators, subelt_type,
1749 elm_item, LIR_OprFact::intConst(sub_offset), result,
1750 NULL, NULL);
1751
1752 if (field->is_null_free()) {
1753 assert(field->type()->is_loaded(), "Must be");
1754 assert(field->type()->is_inlinetype(), "Must be if loaded");
1755 assert(field->type()->as_inline_klass()->is_initialized(), "Must be");
1756 LabelObj* L_end = new LabelObj();
1757 __ cmp(lir_cond_notEqual, result, LIR_OprFact::oopConst(NULL));
1758 __ branch(lir_cond_notEqual, L_end->label());
1759 set_in_conditional_code(true);
1760 Constant* default_value = new Constant(new InstanceConstant(field->type()->as_inline_klass()->default_instance()));
1761 if (default_value->is_pinned()) {
1762 __ move(LIR_OprFact::value_type(default_value->type()), result);
1763 } else {
1764 __ move(load_constant(default_value), result);
1765 }
1766 __ branch_destination(L_end->label());
1767 set_in_conditional_code(false);
1768 }
1769 }
1770
1771 void LIRGenerator::access_flattened_array(bool is_load, LIRItem& array, LIRItem& index, LIRItem& obj_item,
1772 ciField* field, int sub_offset) {
1773 assert(sub_offset == 0 || field != NULL, "Sanity check");
1774
1775 // Find the starting address of the source (inside the array)
1776 LIR_Opr elm_op = get_and_load_element_address(array, index);
1777
1778 ciInlineKlass* elem_klass = NULL;
1779 if (field != NULL) {
1780 elem_klass = field->type()->as_inline_klass();
1781 } else {
1782 elem_klass = array.value()->declared_type()->as_flat_array_klass()->element_klass()->as_inline_klass();
1783 }
1784 for (int i = 0; i < elem_klass->nof_nonstatic_fields(); i++) {
1785 ciField* inner_field = elem_klass->nonstatic_field_at(i);
1786 assert(!inner_field->is_flattened(), "flattened fields must have been expanded");
1787 int obj_offset = inner_field->offset();
1788 int elm_offset = obj_offset - elem_klass->first_field_offset() + sub_offset; // object header is not stored in array.
1789 BasicType field_type = inner_field->type()->basic_type();
1790
1791 // Types which are smaller than int are still passed in an int register.
1792 BasicType reg_type = field_type;
1793 switch (reg_type) {
1794 case T_BYTE:
1795 case T_BOOLEAN:
1796 case T_SHORT:
1797 case T_CHAR:
1798 reg_type = T_INT;
1799 break;
1800 default:
1801 break;
1802 }
1803
1804 LIR_Opr temp = new_register(reg_type);
1805 TempResolvedAddress* elm_resolved_addr = new TempResolvedAddress(as_ValueType(field_type), elm_op);
1806 LIRItem elm_item(elm_resolved_addr, this);
1807
1808 DecoratorSet decorators = IN_HEAP;
1809 if (is_load) {
1810 access_load_at(decorators, field_type,
1811 elm_item, LIR_OprFact::intConst(elm_offset), temp,
1812 NULL, NULL);
1813 access_store_at(decorators, field_type,
1814 obj_item, LIR_OprFact::intConst(obj_offset), temp,
1815 NULL, NULL);
1816 } else {
1817 access_load_at(decorators, field_type,
1818 obj_item, LIR_OprFact::intConst(obj_offset), temp,
1819 NULL, NULL);
1820 access_store_at(decorators, field_type,
1821 elm_item, LIR_OprFact::intConst(elm_offset), temp,
1822 NULL, NULL);
1823 }
1824 }
1825 }
1826
1827 void LIRGenerator::check_flattened_array(LIR_Opr array, LIR_Opr value, CodeStub* slow_path) {
1828 LIR_Opr tmp = new_register(T_METADATA);
1829 __ check_flattened_array(array, value, tmp, slow_path);
1830 }
1831
1832 void LIRGenerator::check_null_free_array(LIRItem& array, LIRItem& value, CodeEmitInfo* info) {
1833 LabelObj* L_end = new LabelObj();
1834 LIR_Opr tmp = new_register(T_METADATA);
1835 __ check_null_free_array(array.result(), tmp);
1836 __ branch(lir_cond_equal, L_end->label());
1837 __ null_check(value.result(), info);
1838 __ branch_destination(L_end->label());
1839 }
1840
1841 bool LIRGenerator::needs_flattened_array_store_check(StoreIndexed* x) {
1842 if (x->elt_type() == T_OBJECT && x->array()->maybe_flattened_array()) {
1843 ciType* type = x->value()->declared_type();
1844 if (type != NULL && type->is_klass()) {
1845 ciKlass* klass = type->as_klass();
1846 if (!klass->can_be_inline_klass() || (klass->is_inlinetype() && !klass->as_inline_klass()->flatten_array())) {
1847 // This is known to be a non-flattened object. If the array is flattened,
1848 // it will be caught by the code generated by array_store_check().
1849 return false;
1850 }
1851 }
1852 // We're not 100% sure, so let's do the flattened_array_store_check.
1853 return true;
1854 }
1855 return false;
1856 }
1857
1858 bool LIRGenerator::needs_null_free_array_store_check(StoreIndexed* x) {
1859 return x->elt_type() == T_OBJECT && x->array()->maybe_null_free_array();
1860 }
1861
1862 void LIRGenerator::do_StoreIndexed(StoreIndexed* x) {
1863 assert(x->is_pinned(),"");
1864 assert(x->elt_type() != T_ARRAY, "never used");
1865 bool is_loaded_flattened_array = x->array()->is_loaded_flattened_array();
1866 bool needs_range_check = x->compute_needs_range_check();
1867 bool use_length = x->length() != NULL;
1868 bool obj_store = is_reference_type(x->elt_type());
1869 bool needs_store_check = obj_store && !(is_loaded_flattened_array && x->is_exact_flattened_array_store()) &&
1870 (x->value()->as_Constant() == NULL ||
1871 !get_jobject_constant(x->value())->is_null_object());
1872
1873 LIRItem array(x->array(), this);
1874 LIRItem index(x->index(), this);
1875 LIRItem value(x->value(), this);
1876 LIRItem length(this);
1877
1878 array.load_item();
1879 index.load_nonconstant();
1880
1881 if (use_length && needs_range_check) {
1882 length.set_instruction(x->length());
1883 length.load_item();
1884 }
1885
1886 if (needs_store_check || x->check_boolean()
1887 || is_loaded_flattened_array || needs_flattened_array_store_check(x) || needs_null_free_array_store_check(x)) {
1888 value.load_item();
1889 } else {
1890 value.load_for_store(x->elt_type());
1891 }
1892
1893 set_no_result(x);
1894
1895 // the CodeEmitInfo must be duplicated for each different
1896 // LIR-instruction because spilling can occur anywhere between two
1897 // instructions and so the debug information must be different
1898 CodeEmitInfo* range_check_info = state_for(x);
1899 CodeEmitInfo* null_check_info = NULL;
1900 if (x->needs_null_check()) {
1901 null_check_info = new CodeEmitInfo(range_check_info);
1902 }
1903
1904 if (GenerateRangeChecks && needs_range_check) {
1905 if (use_length) {
1906 __ cmp(lir_cond_belowEqual, length.result(), index.result());
1907 __ branch(lir_cond_belowEqual, new RangeCheckStub(range_check_info, index.result(), array.result()));
1908 } else {
1909 array_range_check(array.result(), index.result(), null_check_info, range_check_info);
1910 // range_check also does the null check
1911 null_check_info = NULL;
1912 }
1913 }
1914
1915 if (x->should_profile()) {
1916 if (x->array()->is_loaded_flattened_array()) {
1917 // No need to profile a store to a flattened array of known type. This can happen if
1918 // the type only became known after optimizations (for example, after the PhiSimplifier).
1919 x->set_should_profile(false);
1920 } else {
1921 ciMethodData* md = NULL;
1922 ciArrayLoadStoreData* load_store = NULL;
1923 profile_array_type(x, md, load_store);
1924 if (x->array()->maybe_null_free_array()) {
1925 profile_null_free_array(array, md, load_store);
1926 }
1927 profile_element_type(x->value(), md, load_store);
1928 }
1929 }
1930
1931 if (GenerateArrayStoreCheck && needs_store_check) {
1932 CodeEmitInfo* store_check_info = new CodeEmitInfo(range_check_info);
1933 array_store_check(value.result(), array.result(), store_check_info, NULL, -1);
1934 }
1935
1936 if (is_loaded_flattened_array) {
1937 if (!x->value()->is_null_free()) {
1938 __ null_check(value.result(), new CodeEmitInfo(range_check_info));
1939 }
1940 // If array element is an empty inline type, no need to copy anything
1941 if (!x->array()->declared_type()->as_flat_array_klass()->element_klass()->as_inline_klass()->is_empty()) {
1942 access_flattened_array(false, array, index, value);
1943 }
1944 } else {
1945 StoreFlattenedArrayStub* slow_path = NULL;
1946
1947 if (needs_flattened_array_store_check(x)) {
1948 // Check if we indeed have a flattened array
1949 index.load_item();
1950 slow_path = new StoreFlattenedArrayStub(array.result(), index.result(), value.result(), state_for(x, x->state_before()));
1951 check_flattened_array(array.result(), value.result(), slow_path);
1952 set_in_conditional_code(true);
1953 } else if (needs_null_free_array_store_check(x)) {
1954 CodeEmitInfo* info = new CodeEmitInfo(range_check_info);
1955 check_null_free_array(array, value, info);
1956 }
1957
1958 DecoratorSet decorators = IN_HEAP | IS_ARRAY;
1959 if (x->check_boolean()) {
1960 decorators |= C1_MASK_BOOLEAN;
1961 }
1962
1963 access_store_at(decorators, x->elt_type(), array, index.result(), value.result(),
1964 NULL, null_check_info);
1965 if (slow_path != NULL) {
1966 __ branch_destination(slow_path->continuation());
1967 set_in_conditional_code(false);
1968 }
1969 }
1970 }
1971
1972 void LIRGenerator::access_load_at(DecoratorSet decorators, BasicType type,
1973 LIRItem& base, LIR_Opr offset, LIR_Opr result,
1974 CodeEmitInfo* patch_info, CodeEmitInfo* load_emit_info) {
1975 decorators |= ACCESS_READ;
1976 LIRAccess access(this, decorators, base, offset, type, patch_info, load_emit_info);
1977 if (access.is_raw()) {
1978 _barrier_set->BarrierSetC1::load_at(access, result);
1979 } else {
1980 _barrier_set->load_at(access, result);
1981 }
1982 }
1983
1984 void LIRGenerator::access_load(DecoratorSet decorators, BasicType type,
1985 LIR_Opr addr, LIR_Opr result) {
1986 decorators |= ACCESS_READ;
1987 LIRAccess access(this, decorators, LIR_OprFact::illegalOpr, LIR_OprFact::illegalOpr, type);
1988 access.set_resolved_addr(addr);
1989 if (access.is_raw()) {
2030 return _barrier_set->BarrierSetC1::atomic_xchg_at(access, value);
2031 } else {
2032 return _barrier_set->atomic_xchg_at(access, value);
2033 }
2034 }
2035
2036 LIR_Opr LIRGenerator::access_atomic_add_at(DecoratorSet decorators, BasicType type,
2037 LIRItem& base, LIRItem& offset, LIRItem& value) {
2038 decorators |= ACCESS_READ;
2039 decorators |= ACCESS_WRITE;
2040 // Atomic operations are SEQ_CST by default
2041 decorators |= ((decorators & MO_DECORATOR_MASK) == 0) ? MO_SEQ_CST : 0;
2042 LIRAccess access(this, decorators, base, offset, type);
2043 if (access.is_raw()) {
2044 return _barrier_set->BarrierSetC1::atomic_add_at(access, value);
2045 } else {
2046 return _barrier_set->atomic_add_at(access, value);
2047 }
2048 }
2049
2050 bool LIRGenerator::inline_type_field_access_prolog(AccessField* x) {
2051 ciField* field = x->field();
2052 assert(!field->is_flattened(), "Flattened field access should have been expanded");
2053 if (!field->is_null_free()) {
2054 return true; // Not an inline type field
2055 }
2056 // Deoptimize if the access is non-static and requires patching (holder not loaded
2057 // or not accessible) because then we only have partial field information and the
2058 // field could be flattened (see ciField constructor).
2059 bool could_be_flat = !x->is_static() && x->needs_patching();
2060 // Deoptimize if we load from a static field with an uninitialized type because we
2061 // need to throw an exception if initialization of the type failed.
2062 bool not_initialized = x->is_static() && x->as_LoadField() != NULL &&
2063 !field->type()->as_instance_klass()->is_initialized();
2064 if (could_be_flat || not_initialized) {
2065 CodeEmitInfo* info = state_for(x, x->state_before());
2066 CodeStub* stub = new DeoptimizeStub(new CodeEmitInfo(info),
2067 Deoptimization::Reason_unloaded,
2068 Deoptimization::Action_make_not_entrant);
2069 __ jump(stub);
2070 return false;
2071 }
2072 return true;
2073 }
2074
2075 void LIRGenerator::do_LoadField(LoadField* x) {
2076 bool needs_patching = x->needs_patching();
2077 bool is_volatile = x->field()->is_volatile();
2078 BasicType field_type = x->field_type();
2079
2080 CodeEmitInfo* info = NULL;
2081 if (needs_patching) {
2082 assert(x->explicit_null_check() == NULL, "can't fold null check into patching field access");
2083 info = state_for(x, x->state_before());
2084 } else if (x->needs_null_check()) {
2085 NullCheck* nc = x->explicit_null_check();
2086 if (nc == NULL) {
2087 info = state_for(x);
2088 } else {
2089 info = state_for(nc);
2090 }
2091 }
2092
2093 LIRItem object(x->obj(), this);
2094
2095 object.load_item();
2096
2097 #ifndef PRODUCT
2098 if (PrintNotLoaded && needs_patching) {
2099 tty->print_cr(" ###class not loaded at load_%s bci %d",
2100 x->is_static() ? "static" : "field", x->printable_bci());
2101 }
2102 #endif
2103
2104 if (!inline_type_field_access_prolog(x)) {
2105 // Field load will always deopt due to unloaded field or holder klass
2106 LIR_Opr result = rlock_result(x, field_type);
2107 __ move(LIR_OprFact::oopConst(NULL), result);
2108 return;
2109 }
2110
2111 bool stress_deopt = StressLoopInvariantCodeMotion && info && info->deoptimize_on_exception();
2112 if (x->needs_null_check() &&
2113 (needs_patching ||
2114 MacroAssembler::needs_explicit_null_check(x->offset()) ||
2115 stress_deopt)) {
2116 LIR_Opr obj = object.result();
2117 if (stress_deopt) {
2118 obj = new_register(T_OBJECT);
2119 __ move(LIR_OprFact::oopConst(NULL), obj);
2120 }
2121 // Emit an explicit null check because the offset is too large.
2122 // If the class is not loaded and the object is NULL, we need to deoptimize to throw a
2123 // NoClassDefFoundError in the interpreter instead of an implicit NPE from compiled code.
2124 __ null_check(obj, new CodeEmitInfo(info), /* deoptimize */ needs_patching);
2125 }
2126
2127 DecoratorSet decorators = IN_HEAP;
2128 if (is_volatile) {
2129 decorators |= MO_SEQ_CST;
2130 }
2131 if (needs_patching) {
2132 decorators |= C1_NEEDS_PATCHING;
2133 }
2134
2135 LIR_Opr result = rlock_result(x, field_type);
2136 access_load_at(decorators, field_type,
2137 object, LIR_OprFact::intConst(x->offset()), result,
2138 info ? new CodeEmitInfo(info) : NULL, info);
2139
2140 ciField* field = x->field();
2141 if (field->is_null_free()) {
2142 // Load from non-flattened inline type field requires
2143 // a null check to replace null with the default value.
2144 ciInstanceKlass* holder = field->holder();
2145 if (field->is_static() && holder->is_loaded()) {
2146 ciObject* val = holder->java_mirror()->field_value(field).as_object();
2147 if (!val->is_null_object()) {
2148 // Static field is initialized, we don't need to perform a null check.
2149 return;
2150 }
2151 }
2152 ciInlineKlass* inline_klass = field->type()->as_inline_klass();
2153 if (inline_klass->is_initialized()) {
2154 LabelObj* L_end = new LabelObj();
2155 __ cmp(lir_cond_notEqual, result, LIR_OprFact::oopConst(NULL));
2156 __ branch(lir_cond_notEqual, L_end->label());
2157 set_in_conditional_code(true);
2158 Constant* default_value = new Constant(new InstanceConstant(inline_klass->default_instance()));
2159 if (default_value->is_pinned()) {
2160 __ move(LIR_OprFact::value_type(default_value->type()), result);
2161 } else {
2162 __ move(load_constant(default_value), result);
2163 }
2164 __ branch_destination(L_end->label());
2165 set_in_conditional_code(false);
2166 } else {
2167 __ cmp(lir_cond_equal, result, LIR_OprFact::oopConst(NULL));
2168 __ branch(lir_cond_equal, new DeoptimizeStub(info, Deoptimization::Reason_uninitialized,
2169 Deoptimization::Action_make_not_entrant));
2170 }
2171 }
2172 }
2173
2174 // int/long jdk.internal.util.Preconditions.checkIndex
2175 void LIRGenerator::do_PreconditionsCheckIndex(Intrinsic* x, BasicType type) {
2176 assert(x->number_of_arguments() == 3, "wrong type");
2177 LIRItem index(x->argument_at(0), this);
2178 LIRItem length(x->argument_at(1), this);
2179 LIRItem oobef(x->argument_at(2), this);
2180
2181 index.load_item();
2182 length.load_item();
2183 oobef.load_item();
2184
2185 LIR_Opr result = rlock_result(x);
2186 // x->state() is created from copy_state_for_exception, it does not contains arguments
2187 // we should prepare them before entering into interpreter mode due to deoptimization.
2188 ValueStack* state = x->state();
2189 for (int i = 0; i < x->number_of_arguments(); i++) {
2190 Value arg = x->argument_at(i);
2191 state->push(arg->type(), arg);
2296 __ move(LIR_OprFact::oopConst(NULL), obj);
2297 __ null_check(obj, new CodeEmitInfo(null_check_info));
2298 }
2299 }
2300
2301 if (GenerateRangeChecks && needs_range_check) {
2302 if (StressLoopInvariantCodeMotion && range_check_info->deoptimize_on_exception()) {
2303 __ branch(lir_cond_always, new RangeCheckStub(range_check_info, index.result(), array.result()));
2304 } else if (use_length) {
2305 // TODO: use a (modified) version of array_range_check that does not require a
2306 // constant length to be loaded to a register
2307 __ cmp(lir_cond_belowEqual, length.result(), index.result());
2308 __ branch(lir_cond_belowEqual, new RangeCheckStub(range_check_info, index.result(), array.result()));
2309 } else {
2310 array_range_check(array.result(), index.result(), null_check_info, range_check_info);
2311 // The range check performs the null check, so clear it out for the load
2312 null_check_info = NULL;
2313 }
2314 }
2315
2316 ciMethodData* md = NULL;
2317 ciArrayLoadStoreData* load_store = NULL;
2318 if (x->should_profile()) {
2319 if (x->array()->is_loaded_flattened_array()) {
2320 // No need to profile a load from a flattened array of known type. This can happen if
2321 // the type only became known after optimizations (for example, after the PhiSimplifier).
2322 x->set_should_profile(false);
2323 } else {
2324 profile_array_type(x, md, load_store);
2325 }
2326 }
2327
2328 Value element;
2329 if (x->vt() != NULL) {
2330 assert(x->array()->is_loaded_flattened_array(), "must be");
2331 // Find the destination address (of the NewInlineTypeInstance).
2332 LIRItem obj_item(x->vt(), this);
2333
2334 access_flattened_array(true, array, index, obj_item,
2335 x->delayed() == NULL ? 0 : x->delayed()->field(),
2336 x->delayed() == NULL ? 0 : x->delayed()->offset());
2337 set_no_result(x);
2338 } else if (x->delayed() != NULL) {
2339 assert(x->array()->is_loaded_flattened_array(), "must be");
2340 LIR_Opr result = rlock_result(x, x->delayed()->field()->type()->basic_type());
2341 access_sub_element(array, index, result, x->delayed()->field(), x->delayed()->offset());
2342 } else if (x->array() != NULL && x->array()->is_loaded_flattened_array() &&
2343 x->array()->declared_type()->as_flat_array_klass()->element_klass()->as_inline_klass()->is_initialized() &&
2344 x->array()->declared_type()->as_flat_array_klass()->element_klass()->as_inline_klass()->is_empty()) {
2345 // Load the default instance instead of reading the element
2346 ciInlineKlass* elem_klass = x->array()->declared_type()->as_flat_array_klass()->element_klass()->as_inline_klass();
2347 LIR_Opr result = rlock_result(x, x->elt_type());
2348 assert(elem_klass->is_initialized(), "Must be");
2349 Constant* default_value = new Constant(new InstanceConstant(elem_klass->default_instance()));
2350 if (default_value->is_pinned()) {
2351 __ move(LIR_OprFact::value_type(default_value->type()), result);
2352 } else {
2353 __ move(load_constant(default_value), result);
2354 }
2355 } else {
2356 LIR_Opr result = rlock_result(x, x->elt_type());
2357 LoadFlattenedArrayStub* slow_path = NULL;
2358
2359 if (x->should_profile() && x->array()->maybe_null_free_array()) {
2360 profile_null_free_array(array, md, load_store);
2361 }
2362
2363 if (x->elt_type() == T_OBJECT && x->array()->maybe_flattened_array()) {
2364 assert(x->delayed() == NULL, "Delayed LoadIndexed only apply to loaded_flattened_arrays");
2365 index.load_item();
2366 // if we are loading from flattened array, load it using a runtime call
2367 slow_path = new LoadFlattenedArrayStub(array.result(), index.result(), result, state_for(x, x->state_before()));
2368 check_flattened_array(array.result(), LIR_OprFact::illegalOpr, slow_path);
2369 set_in_conditional_code(true);
2370 }
2371
2372 DecoratorSet decorators = IN_HEAP | IS_ARRAY;
2373 access_load_at(decorators, x->elt_type(),
2374 array, index.result(), result,
2375 NULL, null_check_info);
2376
2377 if (slow_path != NULL) {
2378 __ branch_destination(slow_path->continuation());
2379 set_in_conditional_code(false);
2380 }
2381
2382 element = x;
2383 }
2384
2385 if (x->should_profile()) {
2386 profile_element_type(element, md, load_store);
2387 }
2388 }
2389
2390 void LIRGenerator::do_Deoptimize(Deoptimize* x) {
2391 // This happens only when a class X uses the withfield/aconst_init bytecode
2392 // to refer to an inline class V, where V has not yet been loaded/resolved.
2393 // This is not a common case. Let's just deoptimize.
2394 CodeEmitInfo* info = state_for(x, x->state_before());
2395 CodeStub* stub = new DeoptimizeStub(new CodeEmitInfo(info),
2396 Deoptimization::Reason_unloaded,
2397 Deoptimization::Action_make_not_entrant);
2398 __ jump(stub);
2399 LIR_Opr reg = rlock_result(x, T_OBJECT);
2400 __ move(LIR_OprFact::oopConst(NULL), reg);
2401 }
2402
2403 void LIRGenerator::do_NullCheck(NullCheck* x) {
2404 if (x->can_trap()) {
2405 LIRItem value(x->obj(), this);
2406 value.load_item();
2407 CodeEmitInfo* info = state_for(x);
2408 __ null_check(value.result(), info);
2409 }
2410 }
2411
2412
2413 void LIRGenerator::do_TypeCast(TypeCast* x) {
2414 LIRItem value(x->obj(), this);
2415 value.load_item();
2416 // the result is the same as from the node we are casting
2417 set_result(x, value.result());
2418 }
2419
2420
2421 void LIRGenerator::do_Throw(Throw* x) {
2880 Compilation* comp = Compilation::current();
2881 if (do_update) {
2882 // try to find exact type, using CHA if possible, so that loading
2883 // the klass from the object can be avoided
2884 ciType* type = obj->exact_type();
2885 if (type == NULL) {
2886 type = obj->declared_type();
2887 type = comp->cha_exact_type(type);
2888 }
2889 assert(type == NULL || type->is_klass(), "type should be class");
2890 exact_klass = (type != NULL && type->is_loaded()) ? (ciKlass*)type : NULL;
2891
2892 do_update = exact_klass == NULL || ciTypeEntries::valid_ciklass(profiled_k) != exact_klass;
2893 }
2894
2895 if (!do_null && !do_update) {
2896 return result;
2897 }
2898
2899 ciKlass* exact_signature_k = NULL;
2900 if (do_update && signature_at_call_k != NULL) {
2901 // Is the type from the signature exact (the only one possible)?
2902 exact_signature_k = signature_at_call_k->exact_klass();
2903 if (exact_signature_k == NULL) {
2904 exact_signature_k = comp->cha_exact_type(signature_at_call_k);
2905 } else {
2906 result = exact_signature_k;
2907 // Known statically. No need to emit any code: prevent
2908 // LIR_Assembler::emit_profile_type() from emitting useless code
2909 profiled_k = ciTypeEntries::with_status(result, profiled_k);
2910 }
2911 // exact_klass and exact_signature_k can be both non NULL but
2912 // different if exact_klass is loaded after the ciObject for
2913 // exact_signature_k is created.
2914 if (exact_klass == NULL && exact_signature_k != NULL && exact_klass != exact_signature_k) {
2915 // sometimes the type of the signature is better than the best type
2916 // the compiler has
2917 exact_klass = exact_signature_k;
2918 }
2919 if (callee_signature_k != NULL &&
2920 callee_signature_k != signature_at_call_k) {
2965 assert(!src->is_illegal(), "check");
2966 BasicType t = src->type();
2967 if (is_reference_type(t)) {
2968 intptr_t profiled_k = parameters->type(j);
2969 Local* local = x->state()->local_at(java_index)->as_Local();
2970 ciKlass* exact = profile_type(md, md->byte_offset_of_slot(parameters_type_data, ParametersTypeData::type_offset(0)),
2971 in_bytes(ParametersTypeData::type_offset(j)) - in_bytes(ParametersTypeData::type_offset(0)),
2972 profiled_k, local, mdp, false, local->declared_type()->as_klass(), NULL);
2973 // If the profile is known statically set it once for all and do not emit any code
2974 if (exact != NULL) {
2975 md->set_parameter_type(j, exact);
2976 }
2977 j++;
2978 }
2979 java_index += type2size[t];
2980 }
2981 }
2982 }
2983 }
2984
2985 void LIRGenerator::profile_flags(ciMethodData* md, ciProfileData* data, int flag, LIR_Condition condition) {
2986 assert(md != NULL && data != NULL, "should have been initialized");
2987 LIR_Opr mdp = new_register(T_METADATA);
2988 __ metadata2reg(md->constant_encoding(), mdp);
2989 LIR_Address* addr = new LIR_Address(mdp, md->byte_offset_of_slot(data, DataLayout::flags_offset()), T_BYTE);
2990 LIR_Opr flags = new_register(T_INT);
2991 __ move(addr, flags);
2992 if (condition != lir_cond_always) {
2993 LIR_Opr update = new_register(T_INT);
2994 __ cmove(condition, LIR_OprFact::intConst(0), LIR_OprFact::intConst(flag), update, T_INT);
2995 } else {
2996 __ logical_or(flags, LIR_OprFact::intConst(flag), flags);
2997 }
2998 __ store(flags, addr);
2999 }
3000
3001 void LIRGenerator::profile_null_free_array(LIRItem array, ciMethodData* md, ciArrayLoadStoreData* load_store) {
3002 assert(compilation()->profile_array_accesses(), "array access profiling is disabled");
3003 LabelObj* L_end = new LabelObj();
3004 LIR_Opr tmp = new_register(T_METADATA);
3005 __ check_null_free_array(array.result(), tmp);
3006
3007 profile_flags(md, load_store, ArrayLoadStoreData::null_free_array_byte_constant(), lir_cond_equal);
3008 }
3009
3010 void LIRGenerator::profile_array_type(AccessIndexed* x, ciMethodData*& md, ciArrayLoadStoreData*& load_store) {
3011 assert(compilation()->profile_array_accesses(), "array access profiling is disabled");
3012 int bci = x->profiled_bci();
3013 md = x->profiled_method()->method_data();
3014 assert(md != NULL, "Sanity");
3015 ciProfileData* data = md->bci_to_data(bci);
3016 assert(data != NULL && data->is_ArrayLoadStoreData(), "incorrect profiling entry");
3017 load_store = (ciArrayLoadStoreData*)data;
3018 LIR_Opr mdp = LIR_OprFact::illegalOpr;
3019 profile_type(md, md->byte_offset_of_slot(load_store, ArrayLoadStoreData::array_offset()), 0,
3020 load_store->array()->type(), x->array(), mdp, true, NULL, NULL);
3021 }
3022
3023 void LIRGenerator::profile_element_type(Value element, ciMethodData* md, ciArrayLoadStoreData* load_store) {
3024 assert(compilation()->profile_array_accesses(), "array access profiling is disabled");
3025 assert(md != NULL && load_store != NULL, "should have been initialized");
3026 LIR_Opr mdp = LIR_OprFact::illegalOpr;
3027 profile_type(md, md->byte_offset_of_slot(load_store, ArrayLoadStoreData::element_offset()), 0,
3028 load_store->element()->type(), element, mdp, false, NULL, NULL);
3029 }
3030
3031 void LIRGenerator::do_Base(Base* x) {
3032 __ std_entry(LIR_OprFact::illegalOpr);
3033 // Emit moves from physical registers / stack slots to virtual registers
3034 CallingConvention* args = compilation()->frame_map()->incoming_arguments();
3035 IRScope* irScope = compilation()->hir()->top_scope();
3036 int java_index = 0;
3037 for (int i = 0; i < args->length(); i++) {
3038 LIR_Opr src = args->at(i);
3039 assert(!src->is_illegal(), "check");
3040 BasicType t = src->type();
3041
3042 // Types which are smaller than int are passed as int, so
3043 // correct the type which passed.
3044 switch (t) {
3045 case T_BYTE:
3046 case T_BOOLEAN:
3047 case T_SHORT:
3048 case T_CHAR:
3049 t = T_INT;
3050 break;
3095 LIR_Opr lock = syncLockOpr();
3096 __ load_stack_address_monitor(0, lock);
3097
3098 CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, SynchronizationEntryBCI), NULL, x->check_flag(Instruction::DeoptimizeOnException));
3099 CodeStub* slow_path = new MonitorEnterStub(obj, lock, info);
3100
3101 // receiver is guaranteed non-NULL so don't need CodeEmitInfo
3102 __ lock_object(syncTempOpr(), obj, lock, new_register(T_OBJECT), slow_path, NULL);
3103 }
3104 }
3105 if (compilation()->age_code()) {
3106 CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, 0), NULL, false);
3107 decrement_age(info);
3108 }
3109 // increment invocation counters if needed
3110 if (!method()->is_accessor()) { // Accessors do not have MDOs, so no counting.
3111 profile_parameters(x);
3112 CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, SynchronizationEntryBCI), NULL, false);
3113 increment_invocation_counter(info);
3114 }
3115 if (method()->has_scalarized_args()) {
3116 // Check if deoptimization was triggered (i.e. orig_pc was set) while buffering scalarized inline type arguments
3117 // in the entry point (see comments in frame::deoptimize). If so, deoptimize only now that we have the right state.
3118 CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, 0), NULL, false);
3119 CodeStub* deopt_stub = new DeoptimizeStub(info, Deoptimization::Reason_none, Deoptimization::Action_none);
3120 __ append(new LIR_Op0(lir_check_orig_pc));
3121 __ branch(lir_cond_notEqual, deopt_stub);
3122 }
3123
3124 // all blocks with a successor must end with an unconditional jump
3125 // to the successor even if they are consecutive
3126 __ jump(x->default_sux());
3127 }
3128
3129
3130 void LIRGenerator::do_OsrEntry(OsrEntry* x) {
3131 // construct our frame and model the production of incoming pointer
3132 // to the OSR buffer.
3133 __ osr_entry(LIR_Assembler::osrBufferPointer());
3134 LIR_Opr result = rlock_result(x);
3135 __ move(LIR_Assembler::osrBufferPointer(), result);
3136 }
3137
3138 void LIRGenerator::invoke_load_one_argument(LIRItem* param, LIR_Opr loc) {
3139 if (loc->is_register()) {
3140 param->load_item_force(loc);
3141 } else {
3142 LIR_Address* addr = loc->as_address_ptr();
3143 param->load_for_store(addr->type());
3144 assert(addr->type() != T_PRIMITIVE_OBJECT, "not supported yet");
3145 if (addr->type() == T_OBJECT) {
3146 __ move_wide(param->result(), addr);
3147 } else {
3148 __ move(param->result(), addr);
3149 }
3150 }
3151 }
3152
3153 void LIRGenerator::invoke_load_arguments(Invoke* x, LIRItemList* args, const LIR_OprList* arg_list) {
3154 assert(args->length() == arg_list->length(),
3155 "args=%d, arg_list=%d", args->length(), arg_list->length());
3156 for (int i = x->has_receiver() ? 1 : 0; i < args->length(); i++) {
3157 LIRItem* param = args->at(i);
3158 LIR_Opr loc = arg_list->at(i);
3159 invoke_load_one_argument(param, loc);
3160 }
3161
3162 if (x->has_receiver()) {
3163 LIRItem* receiver = args->at(0);
3164 LIR_Opr loc = arg_list->at(0);
3165 if (loc->is_register()) {
3166 receiver->load_item_force(loc);
3167 } else {
3168 assert(loc->is_address(), "just checking");
3169 receiver->load_for_store(T_OBJECT);
3170 __ move_wide(receiver->result(), loc->as_address_ptr());
3171 }
3172 }
3173 }
3174
3175
3176 // Visits all arguments, returns appropriate items without loading them
3177 LIRItemList* LIRGenerator::invoke_visit_arguments(Invoke* x) {
3178 LIRItemList* argument_items = new LIRItemList();
3179 if (x->has_receiver()) {
3305 __ move(tmp, reg);
3306 }
3307
3308
3309
3310 // Code for : x->x() {x->cond()} x->y() ? x->tval() : x->fval()
3311 void LIRGenerator::do_IfOp(IfOp* x) {
3312 #ifdef ASSERT
3313 {
3314 ValueTag xtag = x->x()->type()->tag();
3315 ValueTag ttag = x->tval()->type()->tag();
3316 assert(xtag == intTag || xtag == objectTag, "cannot handle others");
3317 assert(ttag == addressTag || ttag == intTag || ttag == objectTag || ttag == longTag, "cannot handle others");
3318 assert(ttag == x->fval()->type()->tag(), "cannot handle others");
3319 }
3320 #endif
3321
3322 LIRItem left(x->x(), this);
3323 LIRItem right(x->y(), this);
3324 left.load_item();
3325 if (can_inline_as_constant(right.value()) && !x->substitutability_check()) {
3326 right.dont_load_item();
3327 } else {
3328 // substitutability_check() needs to use right as a base register.
3329 right.load_item();
3330 }
3331
3332 LIRItem t_val(x->tval(), this);
3333 LIRItem f_val(x->fval(), this);
3334 t_val.dont_load_item();
3335 f_val.dont_load_item();
3336
3337 if (x->substitutability_check()) {
3338 substitutability_check(x, left, right, t_val, f_val);
3339 } else {
3340 LIR_Opr reg = rlock_result(x);
3341 __ cmp(lir_cond(x->cond()), left.result(), right.result());
3342 __ cmove(lir_cond(x->cond()), t_val.result(), f_val.result(), reg, as_BasicType(x->x()->type()));
3343 }
3344 }
3345
3346 void LIRGenerator::substitutability_check(IfOp* x, LIRItem& left, LIRItem& right, LIRItem& t_val, LIRItem& f_val) {
3347 assert(x->cond() == If::eql || x->cond() == If::neq, "must be");
3348 bool is_acmpeq = (x->cond() == If::eql);
3349 LIR_Opr equal_result = is_acmpeq ? t_val.result() : f_val.result();
3350 LIR_Opr not_equal_result = is_acmpeq ? f_val.result() : t_val.result();
3351 LIR_Opr result = rlock_result(x);
3352 CodeEmitInfo* info = state_for(x, x->state_before());
3353
3354 substitutability_check_common(x->x(), x->y(), left, right, equal_result, not_equal_result, result, info);
3355 }
3356
3357 void LIRGenerator::substitutability_check(If* x, LIRItem& left, LIRItem& right) {
3358 LIR_Opr equal_result = LIR_OprFact::intConst(1);
3359 LIR_Opr not_equal_result = LIR_OprFact::intConst(0);
3360 LIR_Opr result = new_register(T_INT);
3361 CodeEmitInfo* info = state_for(x, x->state_before());
3362
3363 substitutability_check_common(x->x(), x->y(), left, right, equal_result, not_equal_result, result, info);
3364
3365 assert(x->cond() == If::eql || x->cond() == If::neq, "must be");
3366 __ cmp(lir_cond(x->cond()), result, equal_result);
3367 }
3368
3369 void LIRGenerator::substitutability_check_common(Value left_val, Value right_val, LIRItem& left, LIRItem& right,
3370 LIR_Opr equal_result, LIR_Opr not_equal_result, LIR_Opr result,
3371 CodeEmitInfo* info) {
3372 LIR_Opr tmp1 = LIR_OprFact::illegalOpr;
3373 LIR_Opr tmp2 = LIR_OprFact::illegalOpr;
3374 LIR_Opr left_klass_op = LIR_OprFact::illegalOpr;
3375 LIR_Opr right_klass_op = LIR_OprFact::illegalOpr;
3376
3377 ciKlass* left_klass = left_val ->as_loaded_klass_or_null();
3378 ciKlass* right_klass = right_val->as_loaded_klass_or_null();
3379
3380 if ((left_klass == NULL || right_klass == NULL) ||// The klass is still unloaded, or came from a Phi node.
3381 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
3382 init_temps_for_substitutability_check(tmp1, tmp2);
3383 }
3384
3385 if (left_klass != NULL && left_klass->is_inlinetype() && left_klass == right_klass) {
3386 // No need to load klass -- the operands are statically known to be the same inline klass.
3387 } else {
3388 BasicType t_klass = UseCompressedOops ? T_INT : T_METADATA;
3389 left_klass_op = new_register(t_klass);
3390 right_klass_op = new_register(t_klass);
3391 }
3392
3393 CodeStub* slow_path = new SubstitutabilityCheckStub(left.result(), right.result(), info);
3394 __ substitutability_check(result, left.result(), right.result(), equal_result, not_equal_result,
3395 tmp1, tmp2,
3396 left_klass, right_klass, left_klass_op, right_klass_op, info, slow_path);
3397 }
3398
3399 #ifdef JFR_HAVE_INTRINSICS
3400
3401 void LIRGenerator::do_getEventWriter(Intrinsic* x) {
3402 LabelObj* L_end = new LabelObj();
3403
3404 // FIXME T_ADDRESS should actually be T_METADATA but it can't because the
3405 // meaning of these two is mixed up (see JDK-8026837).
3406 LIR_Address* jobj_addr = new LIR_Address(getThreadPointer(),
3407 in_bytes(THREAD_LOCAL_WRITER_OFFSET_JFR),
3408 T_ADDRESS);
3409 LIR_Opr result = rlock_result(x);
3410 __ move(LIR_OprFact::oopConst(NULL), result);
3411 LIR_Opr jobj = new_register(T_METADATA);
3412 __ move_wide(jobj_addr, jobj);
3413 __ cmp(lir_cond_equal, jobj, LIR_OprFact::metadataConst(0));
3414 __ branch(lir_cond_equal, L_end->label());
3415
3416 access_load(IN_NATIVE, T_OBJECT, LIR_OprFact::address(new LIR_Address(jobj, T_OBJECT)), result);
3672 if (x->recv() != NULL || x->nb_profiled_args() > 0) {
3673 profile_parameters_at_call(x);
3674 }
3675
3676 if (x->recv() != NULL) {
3677 LIRItem value(x->recv(), this);
3678 value.load_item();
3679 recv = new_register(T_OBJECT);
3680 __ move(value.result(), recv);
3681 }
3682 __ profile_call(x->method(), x->bci_of_invoke(), x->callee(), mdo, recv, tmp, x->known_holder());
3683 }
3684
3685 void LIRGenerator::do_ProfileReturnType(ProfileReturnType* x) {
3686 int bci = x->bci_of_invoke();
3687 ciMethodData* md = x->method()->method_data_or_null();
3688 assert(md != NULL, "Sanity");
3689 ciProfileData* data = md->bci_to_data(bci);
3690 if (data != NULL) {
3691 assert(data->is_CallTypeData() || data->is_VirtualCallTypeData(), "wrong profile data type");
3692 ciSingleTypeEntry* ret = data->is_CallTypeData() ? ((ciCallTypeData*)data)->ret() : ((ciVirtualCallTypeData*)data)->ret();
3693 LIR_Opr mdp = LIR_OprFact::illegalOpr;
3694
3695 bool ignored_will_link;
3696 ciSignature* signature_at_call = NULL;
3697 x->method()->get_method_at_bci(bci, ignored_will_link, &signature_at_call);
3698
3699 // The offset within the MDO of the entry to update may be too large
3700 // to be used in load/store instructions on some platforms. So have
3701 // profile_type() compute the address of the profile in a register.
3702 ciKlass* exact = profile_type(md, md->byte_offset_of_slot(data, ret->type_offset()), 0,
3703 ret->type(), x->ret(), mdp,
3704 !x->needs_null_check(),
3705 signature_at_call->return_type()->as_klass(),
3706 x->callee()->signature()->return_type()->as_klass());
3707 if (exact != NULL) {
3708 md->set_return_type(bci, exact);
3709 }
3710 }
3711 }
3712
3713 bool LIRGenerator::profile_inline_klass(ciMethodData* md, ciProfileData* data, Value value, int flag) {
3714 ciKlass* klass = value->as_loaded_klass_or_null();
3715 if (klass != NULL) {
3716 if (klass->is_inlinetype()) {
3717 profile_flags(md, data, flag, lir_cond_always);
3718 } else if (klass->can_be_inline_klass()) {
3719 return false;
3720 }
3721 } else {
3722 return false;
3723 }
3724 return true;
3725 }
3726
3727
3728 void LIRGenerator::do_ProfileACmpTypes(ProfileACmpTypes* x) {
3729 ciMethod* method = x->method();
3730 assert(method != NULL, "method should be set if branch is profiled");
3731 ciMethodData* md = method->method_data_or_null();
3732 assert(md != NULL, "Sanity");
3733 ciProfileData* data = md->bci_to_data(x->bci());
3734 assert(data != NULL, "must have profiling data");
3735 assert(data->is_ACmpData(), "need BranchData for two-way branches");
3736 ciACmpData* acmp = (ciACmpData*)data;
3737 LIR_Opr mdp = LIR_OprFact::illegalOpr;
3738 profile_type(md, md->byte_offset_of_slot(acmp, ACmpData::left_offset()), 0,
3739 acmp->left()->type(), x->left(), mdp, !x->left_maybe_null(), NULL, NULL);
3740 int flags_offset = md->byte_offset_of_slot(data, DataLayout::flags_offset());
3741 if (!profile_inline_klass(md, acmp, x->left(), ACmpData::left_inline_type_byte_constant())) {
3742 LIR_Opr mdp = new_register(T_METADATA);
3743 __ metadata2reg(md->constant_encoding(), mdp);
3744 LIRItem value(x->left(), this);
3745 value.load_item();
3746 __ profile_inline_type(new LIR_Address(mdp, flags_offset, T_INT), value.result(), ACmpData::left_inline_type_byte_constant(), new_register(T_INT), !x->left_maybe_null());
3747 }
3748 profile_type(md, md->byte_offset_of_slot(acmp, ACmpData::left_offset()),
3749 in_bytes(ACmpData::right_offset()) - in_bytes(ACmpData::left_offset()),
3750 acmp->right()->type(), x->right(), mdp, !x->right_maybe_null(), NULL, NULL);
3751 if (!profile_inline_klass(md, acmp, x->right(), ACmpData::right_inline_type_byte_constant())) {
3752 LIR_Opr mdp = new_register(T_METADATA);
3753 __ metadata2reg(md->constant_encoding(), mdp);
3754 LIRItem value(x->right(), this);
3755 value.load_item();
3756 __ profile_inline_type(new LIR_Address(mdp, flags_offset, T_INT), value.result(), ACmpData::right_inline_type_byte_constant(), new_register(T_INT), !x->left_maybe_null());
3757 }
3758 }
3759
3760 void LIRGenerator::do_ProfileInvoke(ProfileInvoke* x) {
3761 // We can safely ignore accessors here, since c2 will inline them anyway,
3762 // accessors are also always mature.
3763 if (!x->inlinee()->is_accessor()) {
3764 CodeEmitInfo* info = state_for(x, x->state(), true);
3765 // Notify the runtime very infrequently only to take care of counter overflows
3766 int freq_log = Tier23InlineeNotifyFreqLog;
3767 double scale;
3768 if (_method->has_option_value(CompileCommand::CompileThresholdScaling, scale)) {
3769 freq_log = CompilerConfig::scaled_freq_log(freq_log, scale);
3770 }
3771 increment_event_counter_impl(info, x->inlinee(), LIR_OprFact::intConst(InvocationCounter::count_increment), right_n_bits(freq_log), InvocationEntryBci, false, true);
3772 }
3773 }
3774
3775 void LIRGenerator::increment_backedge_counter_conditionally(LIR_Condition cond, LIR_Opr left, LIR_Opr right, CodeEmitInfo* info, int left_bci, int right_bci, int bci) {
3776 if (compilation()->is_profiling()) {
3777 #if defined(X86) && !defined(_LP64)
3778 // BEWARE! On 32-bit x86 cmp clobbers its left argument so we need a temp copy.
3779 LIR_Opr left_copy = new_register(left->type());
|