28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "memory/allocation.inline.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "oops/objArrayKlass.hpp"
33 #include "opto/addnode.hpp"
34 #include "opto/arraycopynode.hpp"
35 #include "opto/cfgnode.hpp"
36 #include "opto/compile.hpp"
37 #include "opto/connode.hpp"
38 #include "opto/convertnode.hpp"
39 #include "opto/loopnode.hpp"
40 #include "opto/machnode.hpp"
41 #include "opto/matcher.hpp"
42 #include "opto/memnode.hpp"
43 #include "opto/mulnode.hpp"
44 #include "opto/narrowptrnode.hpp"
45 #include "opto/phaseX.hpp"
46 #include "opto/regmask.hpp"
47 #include "opto/rootnode.hpp"
48 #include "utilities/align.hpp"
49 #include "utilities/copy.hpp"
50 #include "utilities/macros.hpp"
51 #include "utilities/vmError.hpp"
52 #if INCLUDE_ZGC
53 #include "gc/z/c2/zBarrierSetC2.hpp"
54 #endif
55
56 // Portions of code courtesy of Clifford Click
57
58 // Optimization - Graph Style
59
60 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
61
62 //=============================================================================
63 uint MemNode::size_of() const { return sizeof(*this); }
64
65 const TypePtr *MemNode::adr_type() const {
66 Node* adr = in(Address);
67 if (adr == NULL) return NULL; // node is dead
222 // clone the Phi with our address type
223 result = mphi->split_out_instance(t_adr, igvn);
224 } else {
225 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
226 }
227 }
228 return result;
229 }
230
231 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
232 uint alias_idx = phase->C->get_alias_index(tp);
233 Node *mem = mmem;
234 #ifdef ASSERT
235 {
236 // Check that current type is consistent with the alias index used during graph construction
237 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
238 bool consistent = adr_check == NULL || adr_check->empty() ||
239 phase->C->must_alias(adr_check, alias_idx );
240 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
241 if( !consistent && adr_check != NULL && !adr_check->empty() &&
242 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
243 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
244 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
245 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
246 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
247 // don't assert if it is dead code.
248 consistent = true;
249 }
250 if( !consistent ) {
251 st->print("alias_idx==%d, adr_check==", alias_idx);
252 if( adr_check == NULL ) {
253 st->print("NULL");
254 } else {
255 adr_check->dump();
256 }
257 st->cr();
258 print_alias_types();
259 assert(consistent, "adr_check must match alias idx");
260 }
261 }
262 #endif
820 "use LoadKlassNode instead");
821 assert(!(adr_type->isa_aryptr() &&
822 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
823 "use LoadRangeNode instead");
824 // Check control edge of raw loads
825 assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
826 // oop will be recorded in oop map if load crosses safepoint
827 rt->isa_oopptr() || is_immutable_value(adr),
828 "raw memory operations should have control edge");
829 LoadNode* load = NULL;
830 switch (bt) {
831 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
832 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
833 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
834 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
835 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
836 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency); break;
837 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
838 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
839 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
840 case T_OBJECT:
841 #ifdef _LP64
842 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
843 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
844 } else
845 #endif
846 {
847 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
848 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
849 }
850 break;
851 default:
852 ShouldNotReachHere();
853 break;
854 }
855 assert(load != NULL, "LoadNode should have been created");
856 if (unaligned) {
857 load->set_unaligned_access();
858 }
859 if (mismatched) {
1075 // the same pointer-and-offset that we stored to.
1076 // Casted version may carry a dependency and it is respected.
1077 // Thus, we are able to replace L by V.
1078 }
1079 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1080 if (store_Opcode() != st->Opcode())
1081 return NULL;
1082 return st->in(MemNode::ValueIn);
1083 }
1084
1085 // A load from a freshly-created object always returns zero.
1086 // (This can happen after LoadNode::Ideal resets the load's memory input
1087 // to find_captured_store, which returned InitializeNode::zero_memory.)
1088 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1089 (st->in(0) == ld_alloc) &&
1090 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1091 // return a zero value for the load's basic type
1092 // (This is one of the few places where a generic PhaseTransform
1093 // can create new nodes. Think of it as lazily manifesting
1094 // virtually pre-existing constants.)
1095 return phase->zerocon(memory_type());
1096 }
1097
1098 // A load from an initialization barrier can match a captured store.
1099 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1100 InitializeNode* init = st->in(0)->as_Initialize();
1101 AllocateNode* alloc = init->allocation();
1102 if ((alloc != NULL) && (alloc == ld_alloc)) {
1103 // examine a captured store value
1104 st = init->find_captured_store(ld_off, memory_size(), phase);
1105 if (st != NULL) {
1106 continue; // take one more trip around
1107 }
1108 }
1109 }
1110
1111 // Load boxed value from result of valueOf() call is input parameter.
1112 if (this->is_Load() && ld_adr->is_AddP() &&
1113 (tp != NULL) && tp->is_ptr_to_boxed_value()) {
1114 intptr_t ignore = 0;
1132 //----------------------is_instance_field_load_with_local_phi------------------
1133 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1134 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1135 in(Address)->is_AddP() ) {
1136 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1137 // Only instances and boxed values.
1138 if( t_oop != NULL &&
1139 (t_oop->is_ptr_to_boxed_value() ||
1140 t_oop->is_known_instance_field()) &&
1141 t_oop->offset() != Type::OffsetBot &&
1142 t_oop->offset() != Type::OffsetTop) {
1143 return true;
1144 }
1145 }
1146 return false;
1147 }
1148
1149 //------------------------------Identity---------------------------------------
1150 // Loads are identity if previous store is to same address
1151 Node* LoadNode::Identity(PhaseGVN* phase) {
1152 // If the previous store-maker is the right kind of Store, and the store is
1153 // to the same address, then we are equal to the value stored.
1154 Node* mem = in(Memory);
1155 Node* value = can_see_stored_value(mem, phase);
1156 if( value ) {
1157 // byte, short & char stores truncate naturally.
1158 // A load has to load the truncated value which requires
1159 // some sort of masking operation and that requires an
1160 // Ideal call instead of an Identity call.
1161 if (memory_size() < BytesPerInt) {
1162 // If the input to the store does not fit with the load's result type,
1163 // it must be truncated via an Ideal call.
1164 if (!phase->type(value)->higher_equal(phase->type(this)))
1165 return this;
1166 }
1167 // (This works even when value is a Con, but LoadNode::Value
1168 // usually runs first, producing the singleton type of the Con.)
1169 return value;
1170 }
1171
1667 // fold up, do so.
1668 Node* prev_mem = find_previous_store(phase);
1669 if (prev_mem != NULL) {
1670 Node* value = can_see_arraycopy_value(prev_mem, phase);
1671 if (value != NULL) {
1672 return value;
1673 }
1674 }
1675 // Steps (a), (b): Walk past independent stores to find an exact match.
1676 if (prev_mem != NULL && prev_mem != in(MemNode::Memory)) {
1677 // (c) See if we can fold up on the spot, but don't fold up here.
1678 // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
1679 // just return a prior value, which is done by Identity calls.
1680 if (can_see_stored_value(prev_mem, phase)) {
1681 // Make ready for step (d):
1682 set_req(MemNode::Memory, prev_mem);
1683 return this;
1684 }
1685 }
1686
1687 return progress ? this : NULL;
1688 }
1689
1690 // Helper to recognize certain Klass fields which are invariant across
1691 // some group of array types (e.g., int[] or all T[] where T < Object).
1692 const Type*
1693 LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
1694 ciKlass* klass) const {
1695 if (tkls->offset() == in_bytes(Klass::modifier_flags_offset())) {
1696 // The field is Klass::_modifier_flags. Return its (constant) value.
1697 // (Folds up the 2nd indirection in aClassConstant.getModifiers().)
1698 assert(this->Opcode() == Op_LoadI, "must load an int from _modifier_flags");
1699 return TypeInt::make(klass->modifier_flags());
1700 }
1701 if (tkls->offset() == in_bytes(Klass::access_flags_offset())) {
1702 // The field is Klass::_access_flags. Return its (constant) value.
1703 // (Folds up the 2nd indirection in Reflection.getClassAccessFlags(aClassConstant).)
1704 assert(this->Opcode() == Op_LoadI, "must load an int from _access_flags");
1705 return TypeInt::make(klass->access_flags());
1706 }
1756 }
1757 }
1758
1759 // Don't do this for integer types. There is only potential profit if
1760 // the element type t is lower than _type; that is, for int types, if _type is
1761 // more restrictive than t. This only happens here if one is short and the other
1762 // char (both 16 bits), and in those cases we've made an intentional decision
1763 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1764 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1765 //
1766 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1767 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
1768 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1769 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
1770 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1771 // In fact, that could have been the original type of p1, and p1 could have
1772 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1773 // expression (LShiftL quux 3) independently optimized to the constant 8.
1774 if ((t->isa_int() == NULL) && (t->isa_long() == NULL)
1775 && (_type->isa_vect() == NULL)
1776 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1777 // t might actually be lower than _type, if _type is a unique
1778 // concrete subclass of abstract class t.
1779 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
1780 const Type* jt = t->join_speculative(_type);
1781 // In any case, do not allow the join, per se, to empty out the type.
1782 if (jt->empty() && !t->empty()) {
1783 // This can happen if a interface-typed array narrows to a class type.
1784 jt = _type;
1785 }
1786 #ifdef ASSERT
1787 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
1788 // The pointers in the autobox arrays are always non-null
1789 Node* base = adr->in(AddPNode::Base);
1790 if ((base != NULL) && base->is_DecodeN()) {
1791 // Get LoadN node which loads IntegerCache.cache field
1792 base = base->in(1);
1793 }
1794 if ((base != NULL) && base->is_Con()) {
1795 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
1796 if ((base_type != NULL) && base_type->is_autobox_cache()) {
1797 // It could be narrow oop
1798 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
1799 }
1800 }
1801 }
1802 #endif
1803 return jt;
1804 }
1805 }
1806 } else if (tp->base() == Type::InstPtr) {
1807 assert( off != Type::OffsetBot ||
1808 // arrays can be cast to Objects
1809 tp->is_oopptr()->klass()->is_java_lang_Object() ||
1810 // unsafe field access may not have a constant offset
1811 C->has_unsafe_access(),
1812 "Field accesses must be precise" );
1813 // For oop loads, we expect the _type to be precise.
1814
1815 // Optimize loads from constant fields.
1816 const TypeInstPtr* tinst = tp->is_instptr();
1817 ciObject* const_oop = tinst->const_oop();
1818 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) {
1819 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());
1820 if (con_type != NULL) {
1821 return con_type;
1822 }
1823 }
1824 } else if (tp->base() == Type::KlassPtr) {
1825 assert( off != Type::OffsetBot ||
1826 // arrays can be cast to Objects
1827 tp->is_klassptr()->klass()->is_java_lang_Object() ||
1828 // also allow array-loading from the primary supertype
1829 // array during subtype checks
1830 Opcode() == Op_LoadKlass,
1831 "Field accesses must be precise" );
1832 // For klass/static loads, we expect the _type to be precise
1833 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
1834 /* With mirrors being an indirect in the Klass*
1835 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
1836 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
1837 *
1838 * So check the type and klass of the node before the LoadP.
1839 */
1840 Node* adr2 = adr->in(MemNode::Address);
1841 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
1842 if (tkls != NULL && !StressReflectiveCode) {
1843 ciKlass* klass = tkls->klass();
1844 if (klass->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
1845 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1846 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1847 return TypeInstPtr::make(klass->java_mirror());
1848 }
1849 }
1850 }
1851
1852 const TypeKlassPtr *tkls = tp->isa_klassptr();
1853 if (tkls != NULL && !StressReflectiveCode) {
1854 ciKlass* klass = tkls->klass();
1855 if (klass->is_loaded() && tkls->klass_is_exact()) {
1856 // We are loading a field from a Klass metaobject whose identity
1857 // is known at compile time (the type is "exact" or "precise").
1858 // Check for fields we know are maintained as constants by the VM.
1859 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
1860 // The field is Klass::_super_check_offset. Return its (constant) value.
1861 // (Folds up type checking code.)
1862 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
1863 return TypeInt::make(klass->super_check_offset());
1864 }
1865 // Compute index into primary_supers array
1866 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
1867 // Check for overflowing; use unsigned compare to handle the negative case.
1868 if( depth < ciKlass::primary_super_limit() ) {
1869 // The field is an element of Klass::_primary_supers. Return its (constant) value.
1870 // (Folds up type checking code.)
1871 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
1872 ciKlass *ss = klass->super_of_depth(depth);
1873 return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR;
1874 }
1875 const Type* aift = load_array_final_field(tkls, klass);
1876 if (aift != NULL) return aift;
1877 }
1878
1879 // We can still check if we are loading from the primary_supers array at a
1880 // shallow enough depth. Even though the klass is not exact, entries less
1881 // than or equal to its super depth are correct.
1882 if (klass->is_loaded() ) {
1883 ciType *inner = klass;
1884 while( inner->is_obj_array_klass() )
1885 inner = inner->as_obj_array_klass()->base_element_type();
1886 if( inner->is_instance_klass() &&
1887 !inner->as_instance_klass()->flags().is_interface() ) {
1888 // Compute index into primary_supers array
1889 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
1890 // Check for overflowing; use unsigned compare to handle the negative case.
1891 if( depth < ciKlass::primary_super_limit() &&
1892 depth <= klass->super_depth() ) { // allow self-depth checks to handle self-check case
1893 // The field is an element of Klass::_primary_supers. Return its (constant) value.
1894 // (Folds up type checking code.)
1895 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
1896 ciKlass *ss = klass->super_of_depth(depth);
1897 return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR;
1898 }
1899 }
1900 }
1901
1902 // If the type is enough to determine that the thing is not an array,
2108 // Either input is TOP ==> the result is TOP
2109 const Type *t1 = phase->type( in(MemNode::Memory) );
2110 if (t1 == Type::TOP) return Type::TOP;
2111 Node *adr = in(MemNode::Address);
2112 const Type *t2 = phase->type( adr );
2113 if (t2 == Type::TOP) return Type::TOP;
2114 const TypePtr *tp = t2->is_ptr();
2115 if (TypePtr::above_centerline(tp->ptr()) ||
2116 tp->ptr() == TypePtr::Null) return Type::TOP;
2117
2118 // Return a more precise klass, if possible
2119 const TypeInstPtr *tinst = tp->isa_instptr();
2120 if (tinst != NULL) {
2121 ciInstanceKlass* ik = tinst->klass()->as_instance_klass();
2122 int offset = tinst->offset();
2123 if (ik == phase->C->env()->Class_klass()
2124 && (offset == java_lang_Class::klass_offset_in_bytes() ||
2125 offset == java_lang_Class::array_klass_offset_in_bytes())) {
2126 // We are loading a special hidden field from a Class mirror object,
2127 // the field which points to the VM's Klass metaobject.
2128 ciType* t = tinst->java_mirror_type();
2129 // java_mirror_type returns non-null for compile-time Class constants.
2130 if (t != NULL) {
2131 // constant oop => constant klass
2132 if (offset == java_lang_Class::array_klass_offset_in_bytes()) {
2133 if (t->is_void()) {
2134 // We cannot create a void array. Since void is a primitive type return null
2135 // klass. Users of this result need to do a null check on the returned klass.
2136 return TypePtr::NULL_PTR;
2137 }
2138 return TypeKlassPtr::make(ciArrayKlass::make(t));
2139 }
2140 if (!t->is_klass()) {
2141 // a primitive Class (e.g., int.class) has NULL for a klass field
2142 return TypePtr::NULL_PTR;
2143 }
2144 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2145 return TypeKlassPtr::make(t->as_klass());
2146 }
2147 // non-constant mirror, so we can't tell what's going on
2148 }
2149 if( !ik->is_loaded() )
2150 return _type; // Bail out if not loaded
2151 if (offset == oopDesc::klass_offset_in_bytes()) {
2152 if (tinst->klass_is_exact()) {
2153 return TypeKlassPtr::make(ik);
2154 }
2155 // See if we can become precise: no subklasses and no interface
2156 // (Note: We need to support verified interfaces.)
2157 if (!ik->is_interface() && !ik->has_subklass()) {
2158 //assert(!UseExactTypes, "this code should be useless with exact types");
2159 // Add a dependence; if any subclass added we need to recompile
2160 if (!ik->is_final()) {
2161 // %%% should use stronger assert_unique_concrete_subtype instead
2162 phase->C->dependencies()->assert_leaf_type(ik);
2163 }
2164 // Return precise klass
2165 return TypeKlassPtr::make(ik);
2166 }
2167
2168 // Return root of possible klass
2169 return TypeKlassPtr::make(TypePtr::NotNull, ik, 0/*offset*/);
2170 }
2171 }
2172
2173 // Check for loading klass from an array
2174 const TypeAryPtr *tary = tp->isa_aryptr();
2175 if( tary != NULL ) {
2176 ciKlass *tary_klass = tary->klass();
2177 if (tary_klass != NULL // can be NULL when at BOTTOM or TOP
2178 && tary->offset() == oopDesc::klass_offset_in_bytes()) {
2179 if (tary->klass_is_exact()) {
2180 return TypeKlassPtr::make(tary_klass);
2181 }
2182 ciArrayKlass *ak = tary->klass()->as_array_klass();
2183 // If the klass is an object array, we defer the question to the
2184 // array component klass.
2185 if( ak->is_obj_array_klass() ) {
2186 assert( ak->is_loaded(), "" );
2187 ciKlass *base_k = ak->as_obj_array_klass()->base_element_klass();
2188 if( base_k->is_loaded() && base_k->is_instance_klass() ) {
2189 ciInstanceKlass* ik = base_k->as_instance_klass();
2190 // See if we can become precise: no subklasses and no interface
2191 if (!ik->is_interface() && !ik->has_subklass()) {
2192 //assert(!UseExactTypes, "this code should be useless with exact types");
2193 // Add a dependence; if any subclass added we need to recompile
2194 if (!ik->is_final()) {
2195 phase->C->dependencies()->assert_leaf_type(ik);
2196 }
2197 // Return precise array klass
2198 return TypeKlassPtr::make(ak);
2199 }
2200 }
2201 return TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/);
2202 } else { // Found a type-array?
2203 //assert(!UseExactTypes, "this code should be useless with exact types");
2204 assert( ak->is_type_array_klass(), "" );
2205 return TypeKlassPtr::make(ak); // These are always precise
2206 }
2207 }
2208 }
2209
2210 // Check for loading klass from an array klass
2211 const TypeKlassPtr *tkls = tp->isa_klassptr();
2212 if (tkls != NULL && !StressReflectiveCode) {
2213 ciKlass* klass = tkls->klass();
2214 if( !klass->is_loaded() )
2215 return _type; // Bail out if not loaded
2216 if( klass->is_obj_array_klass() &&
2217 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2218 ciKlass* elem = klass->as_obj_array_klass()->element_klass();
2219 // // Always returning precise element type is incorrect,
2220 // // e.g., element type could be object and array may contain strings
2221 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2222
2223 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2224 // according to the element type's subclassing.
2225 return TypeKlassPtr::make(tkls->ptr(), elem, 0/*offset*/);
2226 }
2227 if( klass->is_instance_klass() && tkls->klass_is_exact() &&
2228 tkls->offset() == in_bytes(Klass::super_offset())) {
2229 ciKlass* sup = klass->as_instance_klass()->super();
2230 // The field is Klass::_super. Return its (constant) value.
2231 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2232 return sup ? TypeKlassPtr::make(sup) : TypePtr::NULL_PTR;
2233 }
2234 }
2235
2236 // Bailout case
2237 return LoadNode::Value(phase);
2238 }
2239
2240 //------------------------------Identity---------------------------------------
2241 // To clean up reflective code, simplify k.java_mirror.as_klass to plain k.
2242 // Also feed through the klass in Allocate(...klass...)._klass.
2243 Node* LoadKlassNode::Identity(PhaseGVN* phase) {
2244 return klass_identity_common(phase);
2245 }
2246
2247 Node* LoadNode::klass_identity_common(PhaseGVN* phase) {
2248 Node* x = LoadNode::Identity(phase);
2249 if (x != this) return x;
2250
2251 // Take apart the address into an oop and and offset.
2252 // Return 'this' if we cannot.
2253 Node* adr = in(MemNode::Address);
2254 intptr_t offset = 0;
2255 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2256 if (base == NULL) return this;
2257 const TypeOopPtr* toop = phase->type(adr)->isa_oopptr();
2258 if (toop == NULL) return this;
2259
2260 // Step over potential GC barrier for OopHandle resolve
2261 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
2262 if (bs->is_gc_barrier_node(base)) {
2263 base = bs->step_over_gc_barrier(base);
2264 }
2265
2266 // We can fetch the klass directly through an AllocateNode.
2413 //=============================================================================
2414 //---------------------------StoreNode::make-----------------------------------
2415 // Polymorphic factory method:
2416 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo) {
2417 assert((mo == unordered || mo == release), "unexpected");
2418 Compile* C = gvn.C;
2419 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2420 ctl != NULL, "raw memory operations should have control edge");
2421
2422 switch (bt) {
2423 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2424 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2425 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2426 case T_CHAR:
2427 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2428 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo);
2429 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2430 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo);
2431 case T_METADATA:
2432 case T_ADDRESS:
2433 case T_OBJECT:
2434 #ifdef _LP64
2435 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2436 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2437 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2438 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2439 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2440 adr->bottom_type()->isa_rawptr())) {
2441 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2442 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2443 }
2444 #endif
2445 {
2446 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2447 }
2448 default:
2449 ShouldNotReachHere();
2450 return (StoreNode*)NULL;
2451 }
2452 }
2473 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2474
2475 // Since they are not commoned, do not hash them:
2476 return NO_HASH;
2477 }
2478
2479 //------------------------------Ideal------------------------------------------
2480 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2481 // When a store immediately follows a relevant allocation/initialization,
2482 // try to capture it into the initialization, or hoist it above.
2483 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2484 Node* p = MemNode::Ideal_common(phase, can_reshape);
2485 if (p) return (p == NodeSentinel) ? NULL : p;
2486
2487 Node* mem = in(MemNode::Memory);
2488 Node* address = in(MemNode::Address);
2489 // Back-to-back stores to same address? Fold em up. Generally
2490 // unsafe if I have intervening uses... Also disallowed for StoreCM
2491 // since they must follow each StoreP operation. Redundant StoreCMs
2492 // are eliminated just before matching in final_graph_reshape.
2493 {
2494 Node* st = mem;
2495 // If Store 'st' has more than one use, we cannot fold 'st' away.
2496 // For example, 'st' might be the final state at a conditional
2497 // return. Or, 'st' might be used by some node which is live at
2498 // the same time 'st' is live, which might be unschedulable. So,
2499 // require exactly ONE user until such time as we clone 'mem' for
2500 // each of 'mem's uses (thus making the exactly-1-user-rule hold
2501 // true).
2502 while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2503 // Looking at a dead closed cycle of memory?
2504 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2505 assert(Opcode() == st->Opcode() ||
2506 st->Opcode() == Op_StoreVector ||
2507 Opcode() == Op_StoreVector ||
2508 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2509 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2510 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2511 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2512 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2513
2514 if (st->in(MemNode::Address)->eqv_uncast(address) &&
2515 st->as_Store()->memory_size() <= this->memory_size()) {
2516 Node* use = st->raw_out(0);
2517 phase->igvn_rehash_node_delayed(use);
2518 if (can_reshape) {
2519 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase->is_IterGVN());
2520 } else {
2521 // It's OK to do this in the parser, since DU info is always accurate,
2522 // and the parser always refers to nodes via SafePointNode maps.
2523 use->set_req(MemNode::Memory, st->in(MemNode::Memory));
2524 }
2525 return this;
2526 }
2527 st = st->in(MemNode::Memory);
2528 }
2529 }
2530
2576 // Load then Store? Then the Store is useless
2577 if (val->is_Load() &&
2578 val->in(MemNode::Address)->eqv_uncast(adr) &&
2579 val->in(MemNode::Memory )->eqv_uncast(mem) &&
2580 val->as_Load()->store_Opcode() == Opcode()) {
2581 result = mem;
2582 }
2583
2584 // Two stores in a row of the same value?
2585 if (result == this &&
2586 mem->is_Store() &&
2587 mem->in(MemNode::Address)->eqv_uncast(adr) &&
2588 mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2589 mem->Opcode() == Opcode()) {
2590 result = mem;
2591 }
2592
2593 // Store of zero anywhere into a freshly-allocated object?
2594 // Then the store is useless.
2595 // (It must already have been captured by the InitializeNode.)
2596 if (result == this &&
2597 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
2598 // a newly allocated object is already all-zeroes everywhere
2599 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
2600 result = mem;
2601 }
2602
2603 if (result == this) {
2604 // the store may also apply to zero-bits in an earlier object
2605 Node* prev_mem = find_previous_store(phase);
2606 // Steps (a), (b): Walk past independent stores to find an exact match.
2607 if (prev_mem != NULL) {
2608 Node* prev_val = can_see_stored_value(prev_mem, phase);
2609 if (prev_val != NULL && phase->eqv(prev_val, val)) {
2610 // prev_val and val might differ by a cast; it would be good
2611 // to keep the more informative of the two.
2612 result = mem;
2613 }
2614 }
2615 }
2616 }
2617
2618 if (result != this && phase->is_IterGVN() != NULL) {
2619 MemBarNode* trailing = trailing_membar();
2620 if (trailing != NULL) {
2621 #ifdef ASSERT
2622 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
2623 assert(t_oop == NULL || t_oop->is_known_instance_field(), "only for non escaping objects");
2624 #endif
2625 PhaseIterGVN* igvn = phase->is_IterGVN();
2626 trailing->remove(igvn);
2627 }
2628 }
2629
2630 return result;
2631 }
2632
2901 // Clearing a short array is faster with stores
2902 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2903 // Already know this is a large node, do not try to ideal it
2904 if (!IdealizeClearArrayNode || _is_large) return NULL;
2905
2906 const int unit = BytesPerLong;
2907 const TypeX* t = phase->type(in(2))->isa_intptr_t();
2908 if (!t) return NULL;
2909 if (!t->is_con()) return NULL;
2910 intptr_t raw_count = t->get_con();
2911 intptr_t size = raw_count;
2912 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
2913 // Clearing nothing uses the Identity call.
2914 // Negative clears are possible on dead ClearArrays
2915 // (see jck test stmt114.stmt11402.val).
2916 if (size <= 0 || size % unit != 0) return NULL;
2917 intptr_t count = size / unit;
2918 // Length too long; communicate this to matchers and assemblers.
2919 // Assemblers are responsible to produce fast hardware clears for it.
2920 if (size > InitArrayShortSize) {
2921 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
2922 }
2923 Node *mem = in(1);
2924 if( phase->type(mem)==Type::TOP ) return NULL;
2925 Node *adr = in(3);
2926 const Type* at = phase->type(adr);
2927 if( at==Type::TOP ) return NULL;
2928 const TypePtr* atp = at->isa_ptr();
2929 // adjust atp to be the correct array element address type
2930 if (atp == NULL) atp = TypePtr::BOTTOM;
2931 else atp = atp->add_offset(Type::OffsetBot);
2932 // Get base for derived pointer purposes
2933 if( adr->Opcode() != Op_AddP ) Unimplemented();
2934 Node *base = adr->in(1);
2935
2936 Node *zero = phase->makecon(TypeLong::ZERO);
2937 Node *off = phase->MakeConX(BytesPerLong);
2938 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
2939 count--;
2940 while( count-- ) {
2941 mem = phase->transform(mem);
2942 adr = phase->transform(new AddPNode(base,adr,off));
2943 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
2944 }
2945 return mem;
2946 }
2947
2948 //----------------------------step_through----------------------------------
2949 // Return allocation input memory edge if it is different instance
2950 // or itself if it is the one we are looking for.
2951 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseTransform* phase) {
2952 Node* n = *np;
2953 assert(n->is_ClearArray(), "sanity");
2954 intptr_t offset;
2955 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
2956 // This method is called only before Allocate nodes are expanded
2957 // during macro nodes expansion. Before that ClearArray nodes are
2958 // only generated in PhaseMacroExpand::generate_arraycopy() (before
2959 // Allocate nodes are expanded) which follows allocations.
2960 assert(alloc != NULL, "should have allocation");
2961 if (alloc->_idx == instance_id) {
2962 // Can not bypass initialization of the instance we are looking for.
2963 return false;
2964 }
2965 // Otherwise skip it.
2966 InitializeNode* init = alloc->initialization();
2967 if (init != NULL)
2968 *np = init->in(TypeFunc::Memory);
2969 else
2970 *np = alloc->in(TypeFunc::Memory);
2971 return true;
2972 }
2973
2974 //----------------------------clear_memory-------------------------------------
2975 // Generate code to initialize object storage to zero.
2976 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
2977 intptr_t start_offset,
2978 Node* end_offset,
2979 PhaseGVN* phase) {
2980 intptr_t offset = start_offset;
2981
2982 int unit = BytesPerLong;
2983 if ((offset % unit) != 0) {
2984 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
2985 adr = phase->transform(adr);
2986 const TypePtr* atp = TypeRawPtr::BOTTOM;
2987 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
2988 mem = phase->transform(mem);
2989 offset += BytesPerInt;
2990 }
2991 assert((offset % unit) == 0, "");
2992
2993 // Initialize the remaining stuff, if any, with a ClearArray.
2994 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
2995 }
2996
2997 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
2998 Node* start_offset,
2999 Node* end_offset,
3000 PhaseGVN* phase) {
3001 if (start_offset == end_offset) {
3002 // nothing to do
3003 return mem;
3004 }
3005
3006 int unit = BytesPerLong;
3007 Node* zbase = start_offset;
3008 Node* zend = end_offset;
3009
3010 // Scale to the unit required by the CPU:
3011 if (!Matcher::init_array_count_is_in_bytes) {
3012 Node* shift = phase->intcon(exact_log2(unit));
3013 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3014 zend = phase->transform(new URShiftXNode(zend, shift) );
3015 }
3016
3017 // Bulk clear double-words
3018 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3019 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3020 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
3021 return phase->transform(mem);
3022 }
3023
3024 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3025 intptr_t start_offset,
3026 intptr_t end_offset,
3027 PhaseGVN* phase) {
3028 if (start_offset == end_offset) {
3029 // nothing to do
3030 return mem;
3031 }
3032
3033 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3034 intptr_t done_offset = end_offset;
3035 if ((done_offset % BytesPerLong) != 0) {
3036 done_offset -= BytesPerInt;
3037 }
3038 if (done_offset > start_offset) {
3039 mem = clear_memory(ctl, mem, dest,
3040 start_offset, phase->MakeConX(done_offset), phase);
3041 }
3042 if (done_offset < end_offset) { // emit the final 32-bit store
3043 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3044 adr = phase->transform(adr);
3045 const TypePtr* atp = TypeRawPtr::BOTTOM;
3046 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3047 mem = phase->transform(mem);
3048 done_offset += BytesPerInt;
3049 }
3050 assert(done_offset == end_offset, "");
3051 return mem;
3052 }
3053
3054 //=============================================================================
3055 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3056 : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
3057 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3058 #ifdef ASSERT
3059 , _pair_idx(0)
3060 #endif
3061 {
3062 init_class_id(Class_MemBar);
3063 Node* top = C->top();
3064 init_req(TypeFunc::I_O,top);
3065 init_req(TypeFunc::FramePtr,top);
3066 init_req(TypeFunc::ReturnAdr,top);
3165 PhaseIterGVN* igvn = phase->is_IterGVN();
3166 remove(igvn);
3167 // Must return either the original node (now dead) or a new node
3168 // (Do not return a top here, since that would break the uniqueness of top.)
3169 return new ConINode(TypeInt::ZERO);
3170 }
3171 }
3172 return progress ? this : NULL;
3173 }
3174
3175 //------------------------------Value------------------------------------------
3176 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3177 if( !in(0) ) return Type::TOP;
3178 if( phase->type(in(0)) == Type::TOP )
3179 return Type::TOP;
3180 return TypeTuple::MEMBAR;
3181 }
3182
3183 //------------------------------match------------------------------------------
3184 // Construct projections for memory.
3185 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
3186 switch (proj->_con) {
3187 case TypeFunc::Control:
3188 case TypeFunc::Memory:
3189 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3190 }
3191 ShouldNotReachHere();
3192 return NULL;
3193 }
3194
3195 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3196 trailing->_kind = TrailingStore;
3197 leading->_kind = LeadingStore;
3198 #ifdef ASSERT
3199 trailing->_pair_idx = leading->_idx;
3200 leading->_pair_idx = leading->_idx;
3201 #endif
3202 }
3203
3204 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3205 trailing->_kind = TrailingLoadStore;
3451 return (req() > RawStores);
3452 }
3453
3454 void InitializeNode::set_complete(PhaseGVN* phase) {
3455 assert(!is_complete(), "caller responsibility");
3456 _is_complete = Complete;
3457
3458 // After this node is complete, it contains a bunch of
3459 // raw-memory initializations. There is no need for
3460 // it to have anything to do with non-raw memory effects.
3461 // Therefore, tell all non-raw users to re-optimize themselves,
3462 // after skipping the memory effects of this initialization.
3463 PhaseIterGVN* igvn = phase->is_IterGVN();
3464 if (igvn) igvn->add_users_to_worklist(this);
3465 }
3466
3467 // convenience function
3468 // return false if the init contains any stores already
3469 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3470 InitializeNode* init = initialization();
3471 if (init == NULL || init->is_complete()) return false;
3472 init->remove_extra_zeroes();
3473 // for now, if this allocation has already collected any inits, bail:
3474 if (init->is_non_zero()) return false;
3475 init->set_complete(phase);
3476 return true;
3477 }
3478
3479 void InitializeNode::remove_extra_zeroes() {
3480 if (req() == RawStores) return;
3481 Node* zmem = zero_memory();
3482 uint fill = RawStores;
3483 for (uint i = fill; i < req(); i++) {
3484 Node* n = in(i);
3485 if (n->is_top() || n == zmem) continue; // skip
3486 if (fill < i) set_req(fill, n); // compact
3487 ++fill;
3488 }
3489 // delete any empty spaces created:
3490 while (fill < req()) {
3491 del_req(fill);
4195 // z's_done 12 16 16 16 12 16 12
4196 // z's_needed 12 16 16 16 16 16 16
4197 // zsize 0 0 0 0 4 0 4
4198 if (next_full_store < 0) {
4199 // Conservative tack: Zero to end of current word.
4200 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4201 } else {
4202 // Zero to beginning of next fully initialized word.
4203 // Or, don't zero at all, if we are already in that word.
4204 assert(next_full_store >= zeroes_needed, "must go forward");
4205 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4206 zeroes_needed = next_full_store;
4207 }
4208 }
4209
4210 if (zeroes_needed > zeroes_done) {
4211 intptr_t zsize = zeroes_needed - zeroes_done;
4212 // Do some incremental zeroing on rawmem, in parallel with inits.
4213 zeroes_done = align_down(zeroes_done, BytesPerInt);
4214 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4215 zeroes_done, zeroes_needed,
4216 phase);
4217 zeroes_done = zeroes_needed;
4218 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4219 do_zeroing = false; // leave the hole, next time
4220 }
4221 }
4222
4223 // Collect the store and move on:
4224 st->set_req(MemNode::Memory, inits);
4225 inits = st; // put it on the linearized chain
4226 set_req(i, zmem); // unhook from previous position
4227
4228 if (zeroes_done == st_off)
4229 zeroes_done = next_init_off;
4230
4231 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4232
4233 #ifdef ASSERT
4234 // Various order invariants. Weaker than stores_are_sane because
4254 remove_extra_zeroes(); // clear out all the zmems left over
4255 add_req(inits);
4256
4257 if (!(UseTLAB && ZeroTLAB)) {
4258 // If anything remains to be zeroed, zero it all now.
4259 zeroes_done = align_down(zeroes_done, BytesPerInt);
4260 // if it is the last unused 4 bytes of an instance, forget about it
4261 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4262 if (zeroes_done + BytesPerLong >= size_limit) {
4263 AllocateNode* alloc = allocation();
4264 assert(alloc != NULL, "must be present");
4265 if (alloc != NULL && alloc->Opcode() == Op_Allocate) {
4266 Node* klass_node = alloc->in(AllocateNode::KlassNode);
4267 ciKlass* k = phase->type(klass_node)->is_klassptr()->klass();
4268 if (zeroes_done == k->layout_helper())
4269 zeroes_done = size_limit;
4270 }
4271 }
4272 if (zeroes_done < size_limit) {
4273 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4274 zeroes_done, size_in_bytes, phase);
4275 }
4276 }
4277
4278 set_complete(phase);
4279 return rawmem;
4280 }
4281
4282
4283 #ifdef ASSERT
4284 bool InitializeNode::stores_are_sane(PhaseTransform* phase) {
4285 if (is_complete())
4286 return true; // stores could be anything at this point
4287 assert(allocation() != NULL, "must be present");
4288 intptr_t last_off = allocation()->minimum_header_size();
4289 for (uint i = InitializeNode::RawStores; i < req(); i++) {
4290 Node* st = in(i);
4291 intptr_t st_off = get_store_offset(st, phase);
4292 if (st_off < 0) continue; // ignore dead garbage
4293 if (last_off > st_off) {
|
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "memory/allocation.inline.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "oops/objArrayKlass.hpp"
33 #include "opto/addnode.hpp"
34 #include "opto/arraycopynode.hpp"
35 #include "opto/cfgnode.hpp"
36 #include "opto/compile.hpp"
37 #include "opto/connode.hpp"
38 #include "opto/convertnode.hpp"
39 #include "opto/loopnode.hpp"
40 #include "opto/machnode.hpp"
41 #include "opto/matcher.hpp"
42 #include "opto/memnode.hpp"
43 #include "opto/mulnode.hpp"
44 #include "opto/narrowptrnode.hpp"
45 #include "opto/phaseX.hpp"
46 #include "opto/regmask.hpp"
47 #include "opto/rootnode.hpp"
48 #include "opto/valuetypenode.hpp"
49 #include "utilities/align.hpp"
50 #include "utilities/copy.hpp"
51 #include "utilities/macros.hpp"
52 #include "utilities/vmError.hpp"
53 #if INCLUDE_ZGC
54 #include "gc/z/c2/zBarrierSetC2.hpp"
55 #endif
56
57 // Portions of code courtesy of Clifford Click
58
59 // Optimization - Graph Style
60
61 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
62
63 //=============================================================================
64 uint MemNode::size_of() const { return sizeof(*this); }
65
66 const TypePtr *MemNode::adr_type() const {
67 Node* adr = in(Address);
68 if (adr == NULL) return NULL; // node is dead
223 // clone the Phi with our address type
224 result = mphi->split_out_instance(t_adr, igvn);
225 } else {
226 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
227 }
228 }
229 return result;
230 }
231
232 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
233 uint alias_idx = phase->C->get_alias_index(tp);
234 Node *mem = mmem;
235 #ifdef ASSERT
236 {
237 // Check that current type is consistent with the alias index used during graph construction
238 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
239 bool consistent = adr_check == NULL || adr_check->empty() ||
240 phase->C->must_alias(adr_check, alias_idx );
241 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
242 if( !consistent && adr_check != NULL && !adr_check->empty() &&
243 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
244 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
245 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
246 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
247 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
248 // don't assert if it is dead code.
249 consistent = true;
250 }
251 if( !consistent ) {
252 st->print("alias_idx==%d, adr_check==", alias_idx);
253 if( adr_check == NULL ) {
254 st->print("NULL");
255 } else {
256 adr_check->dump();
257 }
258 st->cr();
259 print_alias_types();
260 assert(consistent, "adr_check must match alias idx");
261 }
262 }
263 #endif
821 "use LoadKlassNode instead");
822 assert(!(adr_type->isa_aryptr() &&
823 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
824 "use LoadRangeNode instead");
825 // Check control edge of raw loads
826 assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
827 // oop will be recorded in oop map if load crosses safepoint
828 rt->isa_oopptr() || is_immutable_value(adr),
829 "raw memory operations should have control edge");
830 LoadNode* load = NULL;
831 switch (bt) {
832 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
833 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
834 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
835 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
836 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
837 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency); break;
838 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
839 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
840 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
841 case T_VALUETYPE:
842 case T_OBJECT:
843 #ifdef _LP64
844 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
845 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
846 } else
847 #endif
848 {
849 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
850 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
851 }
852 break;
853 default:
854 ShouldNotReachHere();
855 break;
856 }
857 assert(load != NULL, "LoadNode should have been created");
858 if (unaligned) {
859 load->set_unaligned_access();
860 }
861 if (mismatched) {
1077 // the same pointer-and-offset that we stored to.
1078 // Casted version may carry a dependency and it is respected.
1079 // Thus, we are able to replace L by V.
1080 }
1081 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1082 if (store_Opcode() != st->Opcode())
1083 return NULL;
1084 return st->in(MemNode::ValueIn);
1085 }
1086
1087 // A load from a freshly-created object always returns zero.
1088 // (This can happen after LoadNode::Ideal resets the load's memory input
1089 // to find_captured_store, which returned InitializeNode::zero_memory.)
1090 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1091 (st->in(0) == ld_alloc) &&
1092 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1093 // return a zero value for the load's basic type
1094 // (This is one of the few places where a generic PhaseTransform
1095 // can create new nodes. Think of it as lazily manifesting
1096 // virtually pre-existing constants.)
1097 assert(memory_type() != T_VALUETYPE, "should not be used for value types");
1098 Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
1099 if (default_value != NULL) {
1100 return default_value;
1101 }
1102 assert(ld_alloc->in(AllocateNode::RawDefaultValue) == NULL, "default value may not be null");
1103 return phase->zerocon(memory_type());
1104 }
1105
1106 // A load from an initialization barrier can match a captured store.
1107 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1108 InitializeNode* init = st->in(0)->as_Initialize();
1109 AllocateNode* alloc = init->allocation();
1110 if ((alloc != NULL) && (alloc == ld_alloc)) {
1111 // examine a captured store value
1112 st = init->find_captured_store(ld_off, memory_size(), phase);
1113 if (st != NULL) {
1114 continue; // take one more trip around
1115 }
1116 }
1117 }
1118
1119 // Load boxed value from result of valueOf() call is input parameter.
1120 if (this->is_Load() && ld_adr->is_AddP() &&
1121 (tp != NULL) && tp->is_ptr_to_boxed_value()) {
1122 intptr_t ignore = 0;
1140 //----------------------is_instance_field_load_with_local_phi------------------
1141 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1142 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1143 in(Address)->is_AddP() ) {
1144 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1145 // Only instances and boxed values.
1146 if( t_oop != NULL &&
1147 (t_oop->is_ptr_to_boxed_value() ||
1148 t_oop->is_known_instance_field()) &&
1149 t_oop->offset() != Type::OffsetBot &&
1150 t_oop->offset() != Type::OffsetTop) {
1151 return true;
1152 }
1153 }
1154 return false;
1155 }
1156
1157 //------------------------------Identity---------------------------------------
1158 // Loads are identity if previous store is to same address
1159 Node* LoadNode::Identity(PhaseGVN* phase) {
1160 // Loading from a ValueTypePtr? The ValueTypePtr has the values of
1161 // all fields as input. Look for the field with matching offset.
1162 Node* addr = in(Address);
1163 intptr_t offset;
1164 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
1165 if (base != NULL && base->is_ValueTypePtr() && offset > oopDesc::klass_offset_in_bytes()) {
1166 Node* value = base->as_ValueTypePtr()->field_value_by_offset((int)offset, true);
1167 if (value->is_ValueType()) {
1168 // Non-flattened value type field
1169 ValueTypeNode* vt = value->as_ValueType();
1170 if (vt->is_allocated(phase)) {
1171 value = vt->get_oop();
1172 } else {
1173 // Not yet allocated, bail out
1174 value = NULL;
1175 }
1176 }
1177 if (value != NULL) {
1178 if (Opcode() == Op_LoadN) {
1179 // Encode oop value if we are loading a narrow oop
1180 assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
1181 value = phase->transform(new EncodePNode(value, bottom_type()));
1182 }
1183 return value;
1184 }
1185 }
1186
1187 // If the previous store-maker is the right kind of Store, and the store is
1188 // to the same address, then we are equal to the value stored.
1189 Node* mem = in(Memory);
1190 Node* value = can_see_stored_value(mem, phase);
1191 if( value ) {
1192 // byte, short & char stores truncate naturally.
1193 // A load has to load the truncated value which requires
1194 // some sort of masking operation and that requires an
1195 // Ideal call instead of an Identity call.
1196 if (memory_size() < BytesPerInt) {
1197 // If the input to the store does not fit with the load's result type,
1198 // it must be truncated via an Ideal call.
1199 if (!phase->type(value)->higher_equal(phase->type(this)))
1200 return this;
1201 }
1202 // (This works even when value is a Con, but LoadNode::Value
1203 // usually runs first, producing the singleton type of the Con.)
1204 return value;
1205 }
1206
1702 // fold up, do so.
1703 Node* prev_mem = find_previous_store(phase);
1704 if (prev_mem != NULL) {
1705 Node* value = can_see_arraycopy_value(prev_mem, phase);
1706 if (value != NULL) {
1707 return value;
1708 }
1709 }
1710 // Steps (a), (b): Walk past independent stores to find an exact match.
1711 if (prev_mem != NULL && prev_mem != in(MemNode::Memory)) {
1712 // (c) See if we can fold up on the spot, but don't fold up here.
1713 // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
1714 // just return a prior value, which is done by Identity calls.
1715 if (can_see_stored_value(prev_mem, phase)) {
1716 // Make ready for step (d):
1717 set_req(MemNode::Memory, prev_mem);
1718 return this;
1719 }
1720 }
1721
1722 AllocateNode* alloc = AllocateNode::Ideal_allocation(address, phase);
1723 if (alloc != NULL && mem->is_Proj() &&
1724 mem->in(0) != NULL &&
1725 mem->in(0) == alloc->initialization() &&
1726 Opcode() == Op_LoadX &&
1727 alloc->initialization()->proj_out_or_null(0) != NULL) {
1728 InitializeNode* init = alloc->initialization();
1729 Node* control = init->proj_out(0);
1730 return alloc->make_ideal_mark(phase, address, control, mem, NULL);
1731 }
1732
1733 return progress ? this : NULL;
1734 }
1735
1736 // Helper to recognize certain Klass fields which are invariant across
1737 // some group of array types (e.g., int[] or all T[] where T < Object).
1738 const Type*
1739 LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
1740 ciKlass* klass) const {
1741 if (tkls->offset() == in_bytes(Klass::modifier_flags_offset())) {
1742 // The field is Klass::_modifier_flags. Return its (constant) value.
1743 // (Folds up the 2nd indirection in aClassConstant.getModifiers().)
1744 assert(this->Opcode() == Op_LoadI, "must load an int from _modifier_flags");
1745 return TypeInt::make(klass->modifier_flags());
1746 }
1747 if (tkls->offset() == in_bytes(Klass::access_flags_offset())) {
1748 // The field is Klass::_access_flags. Return its (constant) value.
1749 // (Folds up the 2nd indirection in Reflection.getClassAccessFlags(aClassConstant).)
1750 assert(this->Opcode() == Op_LoadI, "must load an int from _access_flags");
1751 return TypeInt::make(klass->access_flags());
1752 }
1802 }
1803 }
1804
1805 // Don't do this for integer types. There is only potential profit if
1806 // the element type t is lower than _type; that is, for int types, if _type is
1807 // more restrictive than t. This only happens here if one is short and the other
1808 // char (both 16 bits), and in those cases we've made an intentional decision
1809 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1810 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1811 //
1812 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1813 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
1814 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1815 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
1816 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1817 // In fact, that could have been the original type of p1, and p1 could have
1818 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1819 // expression (LShiftL quux 3) independently optimized to the constant 8.
1820 if ((t->isa_int() == NULL) && (t->isa_long() == NULL)
1821 && (_type->isa_vect() == NULL)
1822 && t->isa_valuetype() == NULL
1823 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1824 // t might actually be lower than _type, if _type is a unique
1825 // concrete subclass of abstract class t.
1826 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
1827 const Type* jt = t->join_speculative(_type);
1828 // In any case, do not allow the join, per se, to empty out the type.
1829 if (jt->empty() && !t->empty()) {
1830 // This can happen if a interface-typed array narrows to a class type.
1831 jt = _type;
1832 }
1833 #ifdef ASSERT
1834 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
1835 // The pointers in the autobox arrays are always non-null
1836 Node* base = adr->in(AddPNode::Base);
1837 if ((base != NULL) && base->is_DecodeN()) {
1838 // Get LoadN node which loads IntegerCache.cache field
1839 base = base->in(1);
1840 }
1841 if ((base != NULL) && base->is_Con()) {
1842 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
1843 if ((base_type != NULL) && base_type->is_autobox_cache()) {
1844 // It could be narrow oop
1845 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
1846 }
1847 }
1848 }
1849 #endif
1850 return jt;
1851 }
1852 }
1853 } else if (tp->base() == Type::InstPtr) {
1854 assert( off != Type::OffsetBot ||
1855 // arrays can be cast to Objects
1856 tp->is_oopptr()->klass()->is_java_lang_Object() ||
1857 tp->is_oopptr()->klass() == ciEnv::current()->Class_klass() ||
1858 // unsafe field access may not have a constant offset
1859 C->has_unsafe_access(),
1860 "Field accesses must be precise" );
1861 // For oop loads, we expect the _type to be precise.
1862
1863 const TypeInstPtr* tinst = tp->is_instptr();
1864 BasicType bt = memory_type();
1865
1866 // Fold component and value mirror loads
1867 ciInstanceKlass* ik = tinst->klass()->as_instance_klass();
1868 if (ik == phase->C->env()->Class_klass() && (off == java_lang_Class::component_mirror_offset_in_bytes() ||
1869 off == java_lang_Class::inline_mirror_offset_in_bytes())) {
1870 ciType* mirror_type = tinst->java_mirror_type();
1871 if (mirror_type != NULL) {
1872 const Type* const_oop = TypePtr::NULL_PTR;
1873 if (mirror_type->is_array_klass()) {
1874 const_oop = TypeInstPtr::make(mirror_type->as_array_klass()->component_mirror_instance());
1875 } else if (mirror_type->is_valuetype()) {
1876 const_oop = TypeInstPtr::make(mirror_type->as_value_klass()->inline_mirror_instance());
1877 }
1878 return (bt == T_NARROWOOP) ? const_oop->make_narrowoop() : const_oop;
1879 }
1880 }
1881
1882 // Optimize loads from constant fields.
1883 ciObject* const_oop = tinst->const_oop();
1884 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) {
1885 ciType* mirror_type = const_oop->as_instance()->java_mirror_type();
1886 if (mirror_type != NULL && mirror_type->is_valuetype()) {
1887 ciValueKlass* vk = mirror_type->as_value_klass();
1888 if (off == vk->default_value_offset()) {
1889 // Loading a special hidden field that contains the oop of the default value type
1890 const Type* const_oop = TypeInstPtr::make(vk->default_value_instance());
1891 return (bt == T_NARROWOOP) ? const_oop->make_narrowoop() : const_oop;
1892 }
1893 }
1894 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
1895 if (con_type != NULL) {
1896 return con_type;
1897 }
1898 }
1899 } else if (tp->base() == Type::KlassPtr) {
1900 assert( off != Type::OffsetBot ||
1901 // arrays can be cast to Objects
1902 tp->is_klassptr()->klass() == NULL ||
1903 tp->is_klassptr()->klass()->is_java_lang_Object() ||
1904 // also allow array-loading from the primary supertype
1905 // array during subtype checks
1906 Opcode() == Op_LoadKlass,
1907 "Field accesses must be precise" );
1908 // For klass/static loads, we expect the _type to be precise
1909 } else if (tp->base() == Type::RawPtr && !StressReflectiveCode) {
1910 if (adr->is_Load() && off == 0) {
1911 /* With mirrors being an indirect in the Klass*
1912 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
1913 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
1914 *
1915 * So check the type and klass of the node before the LoadP.
1916 */
1917 Node* adr2 = adr->in(MemNode::Address);
1918 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
1919 if (tkls != NULL) {
1920 ciKlass* klass = tkls->klass();
1921 if (klass != NULL && klass->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
1922 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1923 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1924 return TypeInstPtr::make(klass->java_mirror());
1925 }
1926 }
1927 } else {
1928 // Check for a load of the default value offset from the ValueKlassFixedBlock:
1929 // LoadI(LoadP(value_klass, adr_valueklass_fixed_block_offset), default_value_offset_offset)
1930 intptr_t offset = 0;
1931 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
1932 if (base != NULL && base->is_Load() && offset == in_bytes(ValueKlass::default_value_offset_offset())) {
1933 const TypeKlassPtr* tkls = phase->type(base->in(MemNode::Address))->isa_klassptr();
1934 if (tkls != NULL && tkls->is_loaded() && tkls->klass_is_exact() && tkls->isa_valuetype() &&
1935 tkls->offset() == in_bytes(InstanceKlass::adr_valueklass_fixed_block_offset())) {
1936 assert(base->Opcode() == Op_LoadP, "must load an oop from klass");
1937 assert(Opcode() == Op_LoadI, "must load an int from fixed block");
1938 return TypeInt::make(tkls->klass()->as_value_klass()->default_value_offset());
1939 }
1940 }
1941 }
1942 }
1943
1944 const TypeKlassPtr *tkls = tp->isa_klassptr();
1945 if (tkls != NULL && !StressReflectiveCode) {
1946 ciKlass* klass = tkls->klass();
1947 if (tkls->is_loaded() && tkls->klass_is_exact()) {
1948 // We are loading a field from a Klass metaobject whose identity
1949 // is known at compile time (the type is "exact" or "precise").
1950 // Check for fields we know are maintained as constants by the VM.
1951 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
1952 // The field is Klass::_super_check_offset. Return its (constant) value.
1953 // (Folds up type checking code.)
1954 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
1955 return TypeInt::make(klass->super_check_offset());
1956 }
1957 // Compute index into primary_supers array
1958 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
1959 // Check for overflowing; use unsigned compare to handle the negative case.
1960 if( depth < ciKlass::primary_super_limit() ) {
1961 // The field is an element of Klass::_primary_supers. Return its (constant) value.
1962 // (Folds up type checking code.)
1963 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
1964 ciKlass *ss = klass->super_of_depth(depth);
1965 return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR;
1966 }
1967 const Type* aift = load_array_final_field(tkls, klass);
1968 if (aift != NULL) return aift;
1969 }
1970
1971 // We can still check if we are loading from the primary_supers array at a
1972 // shallow enough depth. Even though the klass is not exact, entries less
1973 // than or equal to its super depth are correct.
1974 if (tkls->is_loaded()) {
1975 ciType *inner = klass;
1976 while( inner->is_obj_array_klass() )
1977 inner = inner->as_obj_array_klass()->base_element_type();
1978 if( inner->is_instance_klass() &&
1979 !inner->as_instance_klass()->flags().is_interface() ) {
1980 // Compute index into primary_supers array
1981 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
1982 // Check for overflowing; use unsigned compare to handle the negative case.
1983 if( depth < ciKlass::primary_super_limit() &&
1984 depth <= klass->super_depth() ) { // allow self-depth checks to handle self-check case
1985 // The field is an element of Klass::_primary_supers. Return its (constant) value.
1986 // (Folds up type checking code.)
1987 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
1988 ciKlass *ss = klass->super_of_depth(depth);
1989 return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR;
1990 }
1991 }
1992 }
1993
1994 // If the type is enough to determine that the thing is not an array,
2200 // Either input is TOP ==> the result is TOP
2201 const Type *t1 = phase->type( in(MemNode::Memory) );
2202 if (t1 == Type::TOP) return Type::TOP;
2203 Node *adr = in(MemNode::Address);
2204 const Type *t2 = phase->type( adr );
2205 if (t2 == Type::TOP) return Type::TOP;
2206 const TypePtr *tp = t2->is_ptr();
2207 if (TypePtr::above_centerline(tp->ptr()) ||
2208 tp->ptr() == TypePtr::Null) return Type::TOP;
2209
2210 // Return a more precise klass, if possible
2211 const TypeInstPtr *tinst = tp->isa_instptr();
2212 if (tinst != NULL) {
2213 ciInstanceKlass* ik = tinst->klass()->as_instance_klass();
2214 int offset = tinst->offset();
2215 if (ik == phase->C->env()->Class_klass()
2216 && (offset == java_lang_Class::klass_offset_in_bytes() ||
2217 offset == java_lang_Class::array_klass_offset_in_bytes())) {
2218 // We are loading a special hidden field from a Class mirror object,
2219 // the field which points to the VM's Klass metaobject.
2220 bool is_indirect_type = true;
2221 ciType* t = tinst->java_mirror_type(&is_indirect_type);
2222 // java_mirror_type returns non-null for compile-time Class constants.
2223 if (t != NULL) {
2224 // constant oop => constant klass
2225 if (offset == java_lang_Class::array_klass_offset_in_bytes()) {
2226 if (t->is_void()) {
2227 // We cannot create a void array. Since void is a primitive type return null
2228 // klass. Users of this result need to do a null check on the returned klass.
2229 return TypePtr::NULL_PTR;
2230 }
2231 return TypeKlassPtr::make(ciArrayKlass::make(t, /* never_null */ !is_indirect_type));
2232 }
2233 if (!t->is_klass()) {
2234 // a primitive Class (e.g., int.class) has NULL for a klass field
2235 return TypePtr::NULL_PTR;
2236 }
2237 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2238 return TypeKlassPtr::make(t->as_klass());
2239 }
2240 // non-constant mirror, so we can't tell what's going on
2241 }
2242 if( !ik->is_loaded() )
2243 return _type; // Bail out if not loaded
2244 if (offset == oopDesc::klass_offset_in_bytes()) {
2245 if (tinst->klass_is_exact()) {
2246 return TypeKlassPtr::make(ik);
2247 }
2248 // See if we can become precise: no subklasses and no interface
2249 // (Note: We need to support verified interfaces.)
2250 if (!ik->is_interface() && !ik->has_subklass()) {
2251 //assert(!UseExactTypes, "this code should be useless with exact types");
2252 // Add a dependence; if any subclass added we need to recompile
2253 if (!ik->is_final()) {
2254 // %%% should use stronger assert_unique_concrete_subtype instead
2255 phase->C->dependencies()->assert_leaf_type(ik);
2256 }
2257 // Return precise klass
2258 return TypeKlassPtr::make(ik);
2259 }
2260
2261 // Return root of possible klass
2262 return TypeKlassPtr::make(TypePtr::NotNull, ik, Type::Offset(0));
2263 }
2264 }
2265
2266 // Check for loading klass from an array
2267 const TypeAryPtr *tary = tp->isa_aryptr();
2268 if (tary != NULL) {
2269 ciKlass *tary_klass = tary->klass();
2270 if (tary_klass != NULL // can be NULL when at BOTTOM or TOP
2271 && tary->offset() == oopDesc::klass_offset_in_bytes()) {
2272 ciArrayKlass* ak = tary_klass->as_array_klass();
2273 // Do not fold klass loads from [V?. The runtime type might be [V due to [V <: [V?
2274 // and the klass for [V is not equal to the klass for [V?.
2275 if (tary->klass_is_exact()) {
2276 return TypeKlassPtr::make(tary_klass);
2277 }
2278
2279 // If the klass is an object array, we defer the question to the
2280 // array component klass.
2281 if (ak->is_obj_array_klass()) {
2282 assert(ak->is_loaded(), "");
2283 ciKlass *base_k = ak->as_obj_array_klass()->base_element_klass();
2284 if (base_k->is_loaded() && base_k->is_instance_klass()) {
2285 ciInstanceKlass *ik = base_k->as_instance_klass();
2286 // See if we can become precise: no subklasses and no interface
2287 if (!ik->is_interface() && !ik->has_subklass() && (!ik->is_valuetype() || ak->storage_properties().is_null_free())) {
2288 //assert(!UseExactTypes, "this code should be useless with exact types");
2289 // Add a dependence; if any subclass added we need to recompile
2290 if (!ik->is_final()) {
2291 phase->C->dependencies()->assert_leaf_type(ik);
2292 }
2293 // Return precise array klass
2294 return TypeKlassPtr::make(ak);
2295 }
2296 }
2297 return TypeKlassPtr::make(TypePtr::NotNull, ak, Type::Offset(0));
2298 } else if (ak->is_type_array_klass()) {
2299 //assert(!UseExactTypes, "this code should be useless with exact types");
2300 return TypeKlassPtr::make(ak); // These are always precise
2301 }
2302 }
2303 }
2304
2305 // Check for loading klass from an array klass
2306 const TypeKlassPtr *tkls = tp->isa_klassptr();
2307 if (tkls != NULL && !StressReflectiveCode) {
2308 if (!tkls->is_loaded()) {
2309 return _type; // Bail out if not loaded
2310 }
2311 ciKlass* klass = tkls->klass();
2312 if( klass->is_obj_array_klass() &&
2313 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2314 ciKlass* elem = klass->as_obj_array_klass()->element_klass();
2315 // // Always returning precise element type is incorrect,
2316 // // e.g., element type could be object and array may contain strings
2317 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2318
2319 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2320 // according to the element type's subclassing.
2321 return TypeKlassPtr::make(tkls->ptr(), elem, Type::Offset(0));
2322 } else if (klass->is_value_array_klass() &&
2323 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2324 ciKlass* elem = klass->as_value_array_klass()->element_klass();
2325 return TypeKlassPtr::make(tkls->ptr(), elem, Type::Offset(0));
2326 }
2327 if( klass->is_instance_klass() && tkls->klass_is_exact() &&
2328 tkls->offset() == in_bytes(Klass::super_offset())) {
2329 ciKlass* sup = klass->as_instance_klass()->super();
2330 // The field is Klass::_super. Return its (constant) value.
2331 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2332 return sup ? TypeKlassPtr::make(sup) : TypePtr::NULL_PTR;
2333 }
2334 }
2335
2336 // Bailout case
2337 return LoadNode::Value(phase);
2338 }
2339
2340 //------------------------------Identity---------------------------------------
2341 // To clean up reflective code, simplify k.java_mirror.as_klass to plain k.
2342 // Also feed through the klass in Allocate(...klass...)._klass.
2343 Node* LoadKlassNode::Identity(PhaseGVN* phase) {
2344 return klass_identity_common(phase);
2345 }
2346
2347 const Type* GetNullFreePropertyNode::Value(PhaseGVN* phase) const {
2348 if (in(1) != NULL) {
2349 const Type* in1_t = phase->type(in(1));
2350 if (in1_t == Type::TOP) {
2351 return Type::TOP;
2352 }
2353 const TypeKlassPtr* tk = in1_t->make_ptr()->is_klassptr();
2354 ciArrayKlass* ak = tk->klass()->as_array_klass();
2355 ciKlass* elem = ak->element_klass();
2356 if (tk->klass_is_exact() || (!elem->is_java_lang_Object() && !elem->is_interface() && !elem->is_valuetype())) {
2357 int props_shift = in1_t->isa_narrowklass() ? oopDesc::narrow_storage_props_shift : oopDesc::wide_storage_props_shift;
2358 ArrayStorageProperties props = ak->storage_properties();
2359 intptr_t storage_properties = props.encode<intptr_t>(props_shift);
2360 if (in1_t->isa_narrowklass()) {
2361 return TypeInt::make((int)storage_properties);
2362 }
2363 return TypeX::make(storage_properties);
2364 }
2365 }
2366 return bottom_type();
2367 }
2368
2369 Node* GetNullFreePropertyNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2370 if (!can_reshape) {
2371 return NULL;
2372 }
2373 if (in(1) != NULL && in(1)->is_Phi()) {
2374 Node* phi = in(1);
2375 Node* r = phi->in(0);
2376 Node* new_phi = new PhiNode(r, bottom_type());
2377 for (uint i = 1; i < r->req(); i++) {
2378 Node* in = phi->in(i);
2379 if (in == NULL) continue;
2380 new_phi->init_req(i, phase->transform(new GetNullFreePropertyNode(in)));
2381 }
2382 return new_phi;
2383 }
2384 return NULL;
2385 }
2386
2387 Node* LoadNode::klass_identity_common(PhaseGVN* phase) {
2388 Node* x = LoadNode::Identity(phase);
2389 if (x != this) return x;
2390
2391 // Take apart the address into an oop and and offset.
2392 // Return 'this' if we cannot.
2393 Node* adr = in(MemNode::Address);
2394 intptr_t offset = 0;
2395 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2396 if (base == NULL) return this;
2397 const TypeOopPtr* toop = phase->type(adr)->isa_oopptr();
2398 if (toop == NULL) return this;
2399
2400 // Step over potential GC barrier for OopHandle resolve
2401 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
2402 if (bs->is_gc_barrier_node(base)) {
2403 base = bs->step_over_gc_barrier(base);
2404 }
2405
2406 // We can fetch the klass directly through an AllocateNode.
2553 //=============================================================================
2554 //---------------------------StoreNode::make-----------------------------------
2555 // Polymorphic factory method:
2556 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo) {
2557 assert((mo == unordered || mo == release), "unexpected");
2558 Compile* C = gvn.C;
2559 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2560 ctl != NULL, "raw memory operations should have control edge");
2561
2562 switch (bt) {
2563 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2564 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2565 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2566 case T_CHAR:
2567 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2568 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo);
2569 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2570 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo);
2571 case T_METADATA:
2572 case T_ADDRESS:
2573 case T_VALUETYPE:
2574 case T_OBJECT:
2575 #ifdef _LP64
2576 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2577 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2578 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2579 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2580 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2581 adr->bottom_type()->isa_rawptr())) {
2582 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2583 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2584 }
2585 #endif
2586 {
2587 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2588 }
2589 default:
2590 ShouldNotReachHere();
2591 return (StoreNode*)NULL;
2592 }
2593 }
2614 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2615
2616 // Since they are not commoned, do not hash them:
2617 return NO_HASH;
2618 }
2619
2620 //------------------------------Ideal------------------------------------------
2621 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2622 // When a store immediately follows a relevant allocation/initialization,
2623 // try to capture it into the initialization, or hoist it above.
2624 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2625 Node* p = MemNode::Ideal_common(phase, can_reshape);
2626 if (p) return (p == NodeSentinel) ? NULL : p;
2627
2628 Node* mem = in(MemNode::Memory);
2629 Node* address = in(MemNode::Address);
2630 // Back-to-back stores to same address? Fold em up. Generally
2631 // unsafe if I have intervening uses... Also disallowed for StoreCM
2632 // since they must follow each StoreP operation. Redundant StoreCMs
2633 // are eliminated just before matching in final_graph_reshape.
2634 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::VALUES) {
2635 Node* st = mem;
2636 // If Store 'st' has more than one use, we cannot fold 'st' away.
2637 // For example, 'st' might be the final state at a conditional
2638 // return. Or, 'st' might be used by some node which is live at
2639 // the same time 'st' is live, which might be unschedulable. So,
2640 // require exactly ONE user until such time as we clone 'mem' for
2641 // each of 'mem's uses (thus making the exactly-1-user-rule hold
2642 // true).
2643 while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2644 // Looking at a dead closed cycle of memory?
2645 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2646 assert(Opcode() == st->Opcode() ||
2647 st->Opcode() == Op_StoreVector ||
2648 Opcode() == Op_StoreVector ||
2649 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2650 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2651 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2652 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
2653 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2654 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2655
2656 if (st->in(MemNode::Address)->eqv_uncast(address) &&
2657 st->as_Store()->memory_size() <= this->memory_size()) {
2658 Node* use = st->raw_out(0);
2659 phase->igvn_rehash_node_delayed(use);
2660 if (can_reshape) {
2661 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase->is_IterGVN());
2662 } else {
2663 // It's OK to do this in the parser, since DU info is always accurate,
2664 // and the parser always refers to nodes via SafePointNode maps.
2665 use->set_req(MemNode::Memory, st->in(MemNode::Memory));
2666 }
2667 return this;
2668 }
2669 st = st->in(MemNode::Memory);
2670 }
2671 }
2672
2718 // Load then Store? Then the Store is useless
2719 if (val->is_Load() &&
2720 val->in(MemNode::Address)->eqv_uncast(adr) &&
2721 val->in(MemNode::Memory )->eqv_uncast(mem) &&
2722 val->as_Load()->store_Opcode() == Opcode()) {
2723 result = mem;
2724 }
2725
2726 // Two stores in a row of the same value?
2727 if (result == this &&
2728 mem->is_Store() &&
2729 mem->in(MemNode::Address)->eqv_uncast(adr) &&
2730 mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2731 mem->Opcode() == Opcode()) {
2732 result = mem;
2733 }
2734
2735 // Store of zero anywhere into a freshly-allocated object?
2736 // Then the store is useless.
2737 // (It must already have been captured by the InitializeNode.)
2738 if (result == this && ReduceFieldZeroing) {
2739 // a newly allocated object is already all-zeroes everywhere
2740 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
2741 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
2742 assert(!phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == NULL, "storing null to value array is forbidden");
2743 result = mem;
2744 }
2745
2746 if (result == this) {
2747 // the store may also apply to zero-bits in an earlier object
2748 Node* prev_mem = find_previous_store(phase);
2749 // Steps (a), (b): Walk past independent stores to find an exact match.
2750 if (prev_mem != NULL) {
2751 Node* prev_val = can_see_stored_value(prev_mem, phase);
2752 if (prev_val != NULL && phase->eqv(prev_val, val)) {
2753 // prev_val and val might differ by a cast; it would be good
2754 // to keep the more informative of the two.
2755 if (phase->type(val)->is_zero_type()) {
2756 result = mem;
2757 } else if (prev_mem->is_Proj() && prev_mem->in(0)->is_Initialize()) {
2758 InitializeNode* init = prev_mem->in(0)->as_Initialize();
2759 AllocateNode* alloc = init->allocation();
2760 if (alloc != NULL && alloc->in(AllocateNode::DefaultValue) == val) {
2761 result = mem;
2762 }
2763 }
2764 }
2765 }
2766 }
2767 }
2768
2769 if (result != this && phase->is_IterGVN() != NULL) {
2770 MemBarNode* trailing = trailing_membar();
2771 if (trailing != NULL) {
2772 #ifdef ASSERT
2773 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
2774 assert(t_oop == NULL || t_oop->is_known_instance_field(), "only for non escaping objects");
2775 #endif
2776 PhaseIterGVN* igvn = phase->is_IterGVN();
2777 trailing->remove(igvn);
2778 }
2779 }
2780
2781 return result;
2782 }
2783
3052 // Clearing a short array is faster with stores
3053 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3054 // Already know this is a large node, do not try to ideal it
3055 if (!IdealizeClearArrayNode || _is_large) return NULL;
3056
3057 const int unit = BytesPerLong;
3058 const TypeX* t = phase->type(in(2))->isa_intptr_t();
3059 if (!t) return NULL;
3060 if (!t->is_con()) return NULL;
3061 intptr_t raw_count = t->get_con();
3062 intptr_t size = raw_count;
3063 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3064 // Clearing nothing uses the Identity call.
3065 // Negative clears are possible on dead ClearArrays
3066 // (see jck test stmt114.stmt11402.val).
3067 if (size <= 0 || size % unit != 0) return NULL;
3068 intptr_t count = size / unit;
3069 // Length too long; communicate this to matchers and assemblers.
3070 // Assemblers are responsible to produce fast hardware clears for it.
3071 if (size > InitArrayShortSize) {
3072 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
3073 }
3074 Node *mem = in(1);
3075 if( phase->type(mem)==Type::TOP ) return NULL;
3076 Node *adr = in(3);
3077 const Type* at = phase->type(adr);
3078 if( at==Type::TOP ) return NULL;
3079 const TypePtr* atp = at->isa_ptr();
3080 // adjust atp to be the correct array element address type
3081 if (atp == NULL) atp = TypePtr::BOTTOM;
3082 else atp = atp->add_offset(Type::OffsetBot);
3083 // Get base for derived pointer purposes
3084 if( adr->Opcode() != Op_AddP ) Unimplemented();
3085 Node *base = adr->in(1);
3086
3087 Node *val = in(4);
3088 Node *off = phase->MakeConX(BytesPerLong);
3089 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3090 count--;
3091 while( count-- ) {
3092 mem = phase->transform(mem);
3093 adr = phase->transform(new AddPNode(base,adr,off));
3094 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3095 }
3096 return mem;
3097 }
3098
3099 //----------------------------step_through----------------------------------
3100 // Return allocation input memory edge if it is different instance
3101 // or itself if it is the one we are looking for.
3102 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseTransform* phase) {
3103 Node* n = *np;
3104 assert(n->is_ClearArray(), "sanity");
3105 intptr_t offset;
3106 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3107 // This method is called only before Allocate nodes are expanded
3108 // during macro nodes expansion. Before that ClearArray nodes are
3109 // only generated in PhaseMacroExpand::generate_arraycopy() (before
3110 // Allocate nodes are expanded) which follows allocations.
3111 assert(alloc != NULL, "should have allocation");
3112 if (alloc->_idx == instance_id) {
3113 // Can not bypass initialization of the instance we are looking for.
3114 return false;
3115 }
3116 // Otherwise skip it.
3117 InitializeNode* init = alloc->initialization();
3118 if (init != NULL)
3119 *np = init->in(TypeFunc::Memory);
3120 else
3121 *np = alloc->in(TypeFunc::Memory);
3122 return true;
3123 }
3124
3125 //----------------------------clear_memory-------------------------------------
3126 // Generate code to initialize object storage to zero.
3127 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3128 Node* val,
3129 Node* raw_val,
3130 intptr_t start_offset,
3131 Node* end_offset,
3132 PhaseGVN* phase) {
3133 intptr_t offset = start_offset;
3134
3135 int unit = BytesPerLong;
3136 if ((offset % unit) != 0) {
3137 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3138 adr = phase->transform(adr);
3139 const TypePtr* atp = TypeRawPtr::BOTTOM;
3140 if (val != NULL) {
3141 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3142 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3143 } else {
3144 assert(raw_val == NULL, "val may not be null");
3145 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3146 }
3147 mem = phase->transform(mem);
3148 offset += BytesPerInt;
3149 }
3150 assert((offset % unit) == 0, "");
3151
3152 // Initialize the remaining stuff, if any, with a ClearArray.
3153 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
3154 }
3155
3156 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3157 Node* raw_val,
3158 Node* start_offset,
3159 Node* end_offset,
3160 PhaseGVN* phase) {
3161 if (start_offset == end_offset) {
3162 // nothing to do
3163 return mem;
3164 }
3165
3166 int unit = BytesPerLong;
3167 Node* zbase = start_offset;
3168 Node* zend = end_offset;
3169
3170 // Scale to the unit required by the CPU:
3171 if (!Matcher::init_array_count_is_in_bytes) {
3172 Node* shift = phase->intcon(exact_log2(unit));
3173 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3174 zend = phase->transform(new URShiftXNode(zend, shift) );
3175 }
3176
3177 // Bulk clear double-words
3178 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3179 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3180 if (raw_val == NULL) {
3181 raw_val = phase->MakeConX(0);
3182 }
3183 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
3184 return phase->transform(mem);
3185 }
3186
3187 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3188 Node* val,
3189 Node* raw_val,
3190 intptr_t start_offset,
3191 intptr_t end_offset,
3192 PhaseGVN* phase) {
3193 if (start_offset == end_offset) {
3194 // nothing to do
3195 return mem;
3196 }
3197
3198 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3199 intptr_t done_offset = end_offset;
3200 if ((done_offset % BytesPerLong) != 0) {
3201 done_offset -= BytesPerInt;
3202 }
3203 if (done_offset > start_offset) {
3204 mem = clear_memory(ctl, mem, dest, val, raw_val,
3205 start_offset, phase->MakeConX(done_offset), phase);
3206 }
3207 if (done_offset < end_offset) { // emit the final 32-bit store
3208 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3209 adr = phase->transform(adr);
3210 const TypePtr* atp = TypeRawPtr::BOTTOM;
3211 if (val != NULL) {
3212 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3213 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3214 } else {
3215 assert(raw_val == NULL, "val may not be null");
3216 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3217 }
3218 mem = phase->transform(mem);
3219 done_offset += BytesPerInt;
3220 }
3221 assert(done_offset == end_offset, "");
3222 return mem;
3223 }
3224
3225 //=============================================================================
3226 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3227 : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
3228 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3229 #ifdef ASSERT
3230 , _pair_idx(0)
3231 #endif
3232 {
3233 init_class_id(Class_MemBar);
3234 Node* top = C->top();
3235 init_req(TypeFunc::I_O,top);
3236 init_req(TypeFunc::FramePtr,top);
3237 init_req(TypeFunc::ReturnAdr,top);
3336 PhaseIterGVN* igvn = phase->is_IterGVN();
3337 remove(igvn);
3338 // Must return either the original node (now dead) or a new node
3339 // (Do not return a top here, since that would break the uniqueness of top.)
3340 return new ConINode(TypeInt::ZERO);
3341 }
3342 }
3343 return progress ? this : NULL;
3344 }
3345
3346 //------------------------------Value------------------------------------------
3347 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3348 if( !in(0) ) return Type::TOP;
3349 if( phase->type(in(0)) == Type::TOP )
3350 return Type::TOP;
3351 return TypeTuple::MEMBAR;
3352 }
3353
3354 //------------------------------match------------------------------------------
3355 // Construct projections for memory.
3356 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
3357 switch (proj->_con) {
3358 case TypeFunc::Control:
3359 case TypeFunc::Memory:
3360 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3361 }
3362 ShouldNotReachHere();
3363 return NULL;
3364 }
3365
3366 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3367 trailing->_kind = TrailingStore;
3368 leading->_kind = LeadingStore;
3369 #ifdef ASSERT
3370 trailing->_pair_idx = leading->_idx;
3371 leading->_pair_idx = leading->_idx;
3372 #endif
3373 }
3374
3375 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3376 trailing->_kind = TrailingLoadStore;
3622 return (req() > RawStores);
3623 }
3624
3625 void InitializeNode::set_complete(PhaseGVN* phase) {
3626 assert(!is_complete(), "caller responsibility");
3627 _is_complete = Complete;
3628
3629 // After this node is complete, it contains a bunch of
3630 // raw-memory initializations. There is no need for
3631 // it to have anything to do with non-raw memory effects.
3632 // Therefore, tell all non-raw users to re-optimize themselves,
3633 // after skipping the memory effects of this initialization.
3634 PhaseIterGVN* igvn = phase->is_IterGVN();
3635 if (igvn) igvn->add_users_to_worklist(this);
3636 }
3637
3638 // convenience function
3639 // return false if the init contains any stores already
3640 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3641 InitializeNode* init = initialization();
3642 if (init == NULL || init->is_complete()) {
3643 return false;
3644 }
3645 init->remove_extra_zeroes();
3646 // for now, if this allocation has already collected any inits, bail:
3647 if (init->is_non_zero()) return false;
3648 init->set_complete(phase);
3649 return true;
3650 }
3651
3652 void InitializeNode::remove_extra_zeroes() {
3653 if (req() == RawStores) return;
3654 Node* zmem = zero_memory();
3655 uint fill = RawStores;
3656 for (uint i = fill; i < req(); i++) {
3657 Node* n = in(i);
3658 if (n->is_top() || n == zmem) continue; // skip
3659 if (fill < i) set_req(fill, n); // compact
3660 ++fill;
3661 }
3662 // delete any empty spaces created:
3663 while (fill < req()) {
3664 del_req(fill);
4368 // z's_done 12 16 16 16 12 16 12
4369 // z's_needed 12 16 16 16 16 16 16
4370 // zsize 0 0 0 0 4 0 4
4371 if (next_full_store < 0) {
4372 // Conservative tack: Zero to end of current word.
4373 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4374 } else {
4375 // Zero to beginning of next fully initialized word.
4376 // Or, don't zero at all, if we are already in that word.
4377 assert(next_full_store >= zeroes_needed, "must go forward");
4378 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4379 zeroes_needed = next_full_store;
4380 }
4381 }
4382
4383 if (zeroes_needed > zeroes_done) {
4384 intptr_t zsize = zeroes_needed - zeroes_done;
4385 // Do some incremental zeroing on rawmem, in parallel with inits.
4386 zeroes_done = align_down(zeroes_done, BytesPerInt);
4387 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4388 allocation()->in(AllocateNode::DefaultValue),
4389 allocation()->in(AllocateNode::RawDefaultValue),
4390 zeroes_done, zeroes_needed,
4391 phase);
4392 zeroes_done = zeroes_needed;
4393 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4394 do_zeroing = false; // leave the hole, next time
4395 }
4396 }
4397
4398 // Collect the store and move on:
4399 st->set_req(MemNode::Memory, inits);
4400 inits = st; // put it on the linearized chain
4401 set_req(i, zmem); // unhook from previous position
4402
4403 if (zeroes_done == st_off)
4404 zeroes_done = next_init_off;
4405
4406 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4407
4408 #ifdef ASSERT
4409 // Various order invariants. Weaker than stores_are_sane because
4429 remove_extra_zeroes(); // clear out all the zmems left over
4430 add_req(inits);
4431
4432 if (!(UseTLAB && ZeroTLAB)) {
4433 // If anything remains to be zeroed, zero it all now.
4434 zeroes_done = align_down(zeroes_done, BytesPerInt);
4435 // if it is the last unused 4 bytes of an instance, forget about it
4436 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4437 if (zeroes_done + BytesPerLong >= size_limit) {
4438 AllocateNode* alloc = allocation();
4439 assert(alloc != NULL, "must be present");
4440 if (alloc != NULL && alloc->Opcode() == Op_Allocate) {
4441 Node* klass_node = alloc->in(AllocateNode::KlassNode);
4442 ciKlass* k = phase->type(klass_node)->is_klassptr()->klass();
4443 if (zeroes_done == k->layout_helper())
4444 zeroes_done = size_limit;
4445 }
4446 }
4447 if (zeroes_done < size_limit) {
4448 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4449 allocation()->in(AllocateNode::DefaultValue),
4450 allocation()->in(AllocateNode::RawDefaultValue),
4451 zeroes_done, size_in_bytes, phase);
4452 }
4453 }
4454
4455 set_complete(phase);
4456 return rawmem;
4457 }
4458
4459
4460 #ifdef ASSERT
4461 bool InitializeNode::stores_are_sane(PhaseTransform* phase) {
4462 if (is_complete())
4463 return true; // stores could be anything at this point
4464 assert(allocation() != NULL, "must be present");
4465 intptr_t last_off = allocation()->minimum_header_size();
4466 for (uint i = InitializeNode::RawStores; i < req(); i++) {
4467 Node* st = in(i);
4468 intptr_t st_off = get_store_offset(st, phase);
4469 if (st_off < 0) continue; // ignore dead garbage
4470 if (last_off > st_off) {
|