64 Node* value, BasicType bt);
65
66 Node* make_leaf_call(Node* ctrl, Node* mem,
67 const TypeFunc* call_type, address call_addr,
68 const char* call_name,
69 const TypePtr* adr_type,
70 Node* parm0 = nullptr, Node* parm1 = nullptr,
71 Node* parm2 = nullptr, Node* parm3 = nullptr,
72 Node* parm4 = nullptr, Node* parm5 = nullptr,
73 Node* parm6 = nullptr, Node* parm7 = nullptr);
74
75 address basictype2arraycopy(BasicType t,
76 Node* src_offset,
77 Node* dest_offset,
78 bool disjoint_bases,
79 const char* &name,
80 bool dest_uninitialized);
81
82 private:
83 // projections extracted from a call node
84 CallProjections _callprojs;
85
86 // Additional data collected during macro expansion
87 bool _has_locks;
88
89 void expand_allocate(AllocateNode *alloc);
90 void expand_allocate_array(AllocateArrayNode *alloc);
91 void expand_allocate_common(AllocateNode* alloc,
92 Node* length,
93 const TypeFunc* slow_call_type,
94 address slow_call_address,
95 Node* valid_length_test);
96 void yank_alloc_node(AllocateNode* alloc);
97 Node *value_from_mem(Node *mem, Node *ctl, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc);
98 Node *value_from_mem_phi(Node *mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc, Node_Stack *value_phis, int level);
99
100 bool eliminate_boxing_node(CallStaticJavaNode *boxing);
101 bool eliminate_allocate_node(AllocateNode *alloc);
102 void undo_previous_scalarizations(GrowableArray <SafePointNode *> safepoints_done, AllocateNode* alloc);
103 bool scalar_replacement(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints);
104 void process_users_of_allocation(CallNode *alloc);
105
106 void eliminate_gc_barrier(Node *p2x);
107 void mark_eliminated_box(Node* box, Node* obj);
108 void mark_eliminated_locking_nodes(AbstractLockNode *alock);
109 bool eliminate_locking_node(AbstractLockNode *alock);
110 void expand_lock_node(LockNode *lock);
111 void expand_unlock_node(UnlockNode *unlock);
112
113 // More helper methods modeled after GraphKit for array copy
114 void insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent = nullptr);
115 Node* array_element_address(Node* ary, Node* idx, BasicType elembt);
116 Node* ConvI2L(Node* offset);
117
118 // helper methods modeled after LibraryCallKit for array copy
119 Node* generate_guard(Node** ctrl, Node* test, RegionNode* region, float true_prob);
120 Node* generate_slow_guard(Node** ctrl, Node* test, RegionNode* region);
121
122 void generate_partial_inlining_block(Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type,
123 RegionNode** exit_block, Node** result_memory, Node* length,
124 Node* src_start, Node* dst_start, BasicType type);
125
126 void generate_negative_guard(Node** ctrl, Node* index, RegionNode* region);
127 void generate_limit_guard(Node** ctrl, Node* offset, Node* subseq_length, Node* array_length, RegionNode* region);
128
129 // More helper methods for array copy
130 Node* generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative);
131 void finish_arraycopy_call(Node* call, Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type);
132 Node* generate_arraycopy(ArrayCopyNode *ac,
133 AllocateArrayNode* alloc,
134 Node** ctrl, MergeMemNode* mem, Node** io,
135 const TypePtr* adr_type,
136 BasicType basic_elem_type,
137 Node* src, Node* src_offset,
138 Node* dest, Node* dest_offset,
139 Node* copy_length,
140 bool disjoint_bases = false,
141 bool length_never_negative = false,
142 RegionNode* slow_region = nullptr);
143 void generate_clear_array(Node* ctrl, MergeMemNode* merge_mem,
144 const TypePtr* adr_type,
145 Node* dest,
146 BasicType basic_elem_type,
147 Node* slice_idx,
148 Node* slice_len,
149 Node* dest_size);
150 bool generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, Node* io,
151 const TypePtr* adr_type,
152 BasicType basic_elem_type,
153 AllocateNode* alloc,
154 Node* src, Node* src_offset,
155 Node* dest, Node* dest_offset,
156 Node* dest_size, bool dest_uninitialized);
157 MergeMemNode* generate_slow_arraycopy(ArrayCopyNode *ac,
158 Node** ctrl, Node* mem, Node** io,
159 const TypePtr* adr_type,
160 Node* src, Node* src_offset,
161 Node* dest, Node* dest_offset,
162 Node* copy_length, bool dest_uninitialized);
163 Node* generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** mem,
164 const TypePtr* adr_type,
165 Node* dest_elem_klass,
166 Node* src, Node* src_offset,
167 Node* dest, Node* dest_offset,
168 Node* copy_length, bool dest_uninitialized);
169 Node* generate_generic_arraycopy(Node** ctrl, MergeMemNode** mem,
170 const TypePtr* adr_type,
171 Node* src, Node* src_offset,
172 Node* dest, Node* dest_offset,
173 Node* copy_length, bool dest_uninitialized);
174 bool generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** mem,
175 const TypePtr* adr_type,
176 BasicType basic_elem_type,
177 bool disjoint_bases,
178 Node* src, Node* src_offset,
179 Node* dest, Node* dest_offset,
180 Node* copy_length, bool dest_uninitialized);
181
182 void expand_arraycopy_node(ArrayCopyNode *ac);
183
184 void expand_subtypecheck_node(SubTypeCheckNode *check);
185
186 int replace_input(Node *use, Node *oldref, Node *newref);
187 void migrate_outs(Node *old, Node *target);
188 Node* opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path = false);
189 void copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call);
190 CallNode* make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, address slow_call,
191 const char* leaf_name, Node* slow_path, Node* parm0, Node* parm1,
192 Node* parm2);
193
194 Node* initialize_object(AllocateNode* alloc,
195 Node* control, Node* rawmem, Node* object,
196 Node* klass_node, Node* length,
197 Node* size_in_bytes);
198
199 Node* make_arraycopy_load(ArrayCopyNode* ac, intptr_t offset, Node* ctl, Node* mem, BasicType ft, const Type *ftype, AllocateNode *alloc);
200
201 public:
202 PhaseMacroExpand(PhaseIterGVN &igvn) : Phase(Macro_Expand), _igvn(igvn), _has_locks(false) {
203 _igvn.set_delay_transform(true);
204 }
205 void eliminate_macro_nodes();
206 bool expand_macro_nodes();
207
208 SafePointScalarObjectNode* create_scalarized_object_description(AllocateNode *alloc, SafePointNode* sfpt);
209 static bool can_eliminate_allocation(PhaseIterGVN *igvn, AllocateNode *alloc, GrowableArray <SafePointNode *> *safepoints);
210
211
212 PhaseIterGVN &igvn() const { return _igvn; }
213
214 #ifndef PRODUCT
215 static int _objs_scalar_replaced_counter;
216 static int _monitor_objects_removed_counter;
217 static int _GC_barriers_removed_counter;
218 static int _memory_barriers_removed_counter;
219 static void print_statistics();
220 static int count_MemBar(Compile *C);
221 #endif
222
223 // Members accessed from BarrierSetC2
224 void replace_node(Node* source, Node* target) { _igvn.replace_node(source, target); }
225 Node* intcon(jint con) const { return _igvn.intcon(con); }
226 Node* longcon(jlong con) const { return _igvn.longcon(con); }
227 Node* makecon(const Type *t) const { return _igvn.makecon(t); }
228 Node* zerocon(BasicType bt) const { return _igvn.zerocon(bt); }
|
64 Node* value, BasicType bt);
65
66 Node* make_leaf_call(Node* ctrl, Node* mem,
67 const TypeFunc* call_type, address call_addr,
68 const char* call_name,
69 const TypePtr* adr_type,
70 Node* parm0 = nullptr, Node* parm1 = nullptr,
71 Node* parm2 = nullptr, Node* parm3 = nullptr,
72 Node* parm4 = nullptr, Node* parm5 = nullptr,
73 Node* parm6 = nullptr, Node* parm7 = nullptr);
74
75 address basictype2arraycopy(BasicType t,
76 Node* src_offset,
77 Node* dest_offset,
78 bool disjoint_bases,
79 const char* &name,
80 bool dest_uninitialized);
81
82 private:
83 // projections extracted from a call node
84 CallProjections* _callprojs;
85
86 // Additional data collected during macro expansion
87 bool _has_locks;
88
89 void expand_allocate(AllocateNode *alloc);
90 void expand_allocate_array(AllocateArrayNode *alloc);
91 void expand_allocate_common(AllocateNode* alloc,
92 Node* length,
93 Node* init_val,
94 const TypeFunc* slow_call_type,
95 address slow_call_address,
96 Node* valid_length_test);
97 void yank_alloc_node(AllocateNode* alloc);
98 Node *value_from_mem(Node *mem, Node *ctl, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc);
99 Node *value_from_mem_phi(Node *mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc, Node_Stack *value_phis, int level);
100 Node* inline_type_from_mem(Node* mem, Node* ctl, ciInlineKlass* vk, const TypeAryPtr* adr_type, int offset, AllocateNode* alloc);
101
102 bool eliminate_boxing_node(CallStaticJavaNode *boxing);
103 bool eliminate_allocate_node(AllocateNode *alloc);
104 void undo_previous_scalarizations(GrowableArray <SafePointNode *> safepoints_done, AllocateNode* alloc);
105 bool scalar_replacement(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints);
106 void process_users_of_allocation(CallNode *alloc, bool inline_alloc = false);
107
108 void eliminate_gc_barrier(Node *p2x);
109 void mark_eliminated_box(Node* box, Node* obj);
110 void mark_eliminated_locking_nodes(AbstractLockNode *alock);
111 bool eliminate_locking_node(AbstractLockNode *alock);
112 void expand_lock_node(LockNode *lock);
113 void expand_unlock_node(UnlockNode *unlock);
114 void expand_mh_intrinsic_return(CallStaticJavaNode* call);
115
116 // More helper methods modeled after GraphKit for array copy
117 void insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent = nullptr);
118 Node* array_element_address(Node* ary, Node* idx, BasicType elembt);
119 Node* ConvI2L(Node* offset);
120
121 // helper methods modeled after LibraryCallKit for array copy
122 Node* generate_guard(Node** ctrl, Node* test, RegionNode* region, float true_prob);
123 Node* generate_slow_guard(Node** ctrl, Node* test, RegionNode* region);
124 Node* generate_fair_guard(Node** ctrl, Node* test, RegionNode* region);
125
126 void generate_partial_inlining_block(Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type,
127 RegionNode** exit_block, Node** result_memory, Node* length,
128 Node* src_start, Node* dst_start, BasicType type);
129
130 void generate_negative_guard(Node** ctrl, Node* index, RegionNode* region);
131 void generate_limit_guard(Node** ctrl, Node* offset, Node* subseq_length, Node* array_length, RegionNode* region);
132
133 // More helper methods for array copy
134 Node* generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative);
135 Node* mark_word_test(Node** ctrl, Node* obj, MergeMemNode* mem, uintptr_t mask_val, RegionNode* region);
136 Node* generate_flat_array_guard(Node** ctrl, Node* array, MergeMemNode* mem, RegionNode* region);
137 Node* generate_null_free_array_guard(Node** ctrl, Node* array, MergeMemNode* mem, RegionNode* region);
138
139 void finish_arraycopy_call(Node* call, Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type);
140 Node* generate_arraycopy(ArrayCopyNode *ac,
141 AllocateArrayNode* alloc,
142 Node** ctrl, MergeMemNode* mem, Node** io,
143 const TypePtr* adr_type,
144 BasicType basic_elem_type,
145 Node* src, Node* src_offset,
146 Node* dest, Node* dest_offset,
147 Node* copy_length,
148 Node* dest_length,
149 bool disjoint_bases = false,
150 bool length_never_negative = false,
151 RegionNode* slow_region = nullptr);
152 void generate_clear_array(Node* ctrl, MergeMemNode* merge_mem,
153 const TypePtr* adr_type,
154 Node* dest,
155 Node* val,
156 Node* raw_val,
157 BasicType basic_elem_type,
158 Node* slice_idx,
159 Node* slice_len,
160 Node* dest_size);
161 bool generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, Node* io,
162 const TypePtr* adr_type,
163 BasicType basic_elem_type,
164 AllocateNode* alloc,
165 Node* src, Node* src_offset,
166 Node* dest, Node* dest_offset,
167 Node* dest_size, bool dest_uninitialized);
168 MergeMemNode* generate_slow_arraycopy(ArrayCopyNode *ac,
169 Node** ctrl, Node* mem, Node** io,
170 const TypePtr* adr_type,
171 Node* src, Node* src_offset,
172 Node* dest, Node* dest_offset,
173 Node* copy_length, bool dest_uninitialized);
174 Node* generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** mem,
175 const TypePtr* adr_type,
176 Node* dest_elem_klass,
177 Node* src, Node* src_offset,
178 Node* dest, Node* dest_offset,
179 Node* copy_length, bool dest_uninitialized);
180 Node* generate_generic_arraycopy(Node** ctrl, MergeMemNode** mem,
181 const TypePtr* adr_type,
182 Node* src, Node* src_offset,
183 Node* dest, Node* dest_offset,
184 Node* copy_length, bool dest_uninitialized);
185 bool generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** mem,
186 const TypePtr* adr_type,
187 BasicType basic_elem_type,
188 bool disjoint_bases,
189 Node* src, Node* src_offset,
190 Node* dest, Node* dest_offset,
191 Node* copy_length, bool dest_uninitialized);
192 const TypePtr* adjust_for_flat_array(const TypeAryPtr* top_dest, Node*& src_offset,
193 Node*& dest_offset, Node*& length, BasicType& dest_elem,
194 Node*& dest_length);
195 void expand_arraycopy_node(ArrayCopyNode *ac);
196
197 void expand_subtypecheck_node(SubTypeCheckNode *check);
198
199 void expand_flatarraycheck_node(FlatArrayCheckNode* check);
200
201 int replace_input(Node *use, Node *oldref, Node *newref);
202 void migrate_outs(Node *old, Node *target);
203 Node* opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path = false);
204 void copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call);
205 CallNode* make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, address slow_call,
206 const char* leaf_name, Node* slow_path, Node* parm0, Node* parm1,
207 Node* parm2);
208
209 Node* initialize_object(AllocateNode* alloc,
210 Node* control, Node* rawmem, Node* object,
211 Node* klass_node, Node* length,
212 Node* size_in_bytes);
213
214 Node* make_arraycopy_load(ArrayCopyNode* ac, intptr_t offset, Node* ctl, Node* mem, BasicType ft, const Type *ftype, AllocateNode *alloc);
215
216 public:
217 PhaseMacroExpand(PhaseIterGVN &igvn) : Phase(Macro_Expand), _igvn(igvn), _has_locks(false) {
218 _igvn.set_delay_transform(true);
219 }
220 void eliminate_macro_nodes();
221 bool expand_macro_nodes();
222
223 SafePointScalarObjectNode* create_scalarized_object_description(AllocateNode *alloc, SafePointNode* sfpt, Unique_Node_List* value_worklist);
224 static bool can_eliminate_allocation(PhaseIterGVN *igvn, AllocateNode *alloc, GrowableArray <SafePointNode *> *safepoints);
225
226
227 PhaseIterGVN &igvn() const { return _igvn; }
228
229 #ifndef PRODUCT
230 static int _objs_scalar_replaced_counter;
231 static int _monitor_objects_removed_counter;
232 static int _GC_barriers_removed_counter;
233 static int _memory_barriers_removed_counter;
234 static void print_statistics();
235 static int count_MemBar(Compile *C);
236 #endif
237
238 // Members accessed from BarrierSetC2
239 void replace_node(Node* source, Node* target) { _igvn.replace_node(source, target); }
240 Node* intcon(jint con) const { return _igvn.intcon(con); }
241 Node* longcon(jlong con) const { return _igvn.longcon(con); }
242 Node* makecon(const Type *t) const { return _igvn.makecon(t); }
243 Node* zerocon(BasicType bt) const { return _igvn.zerocon(bt); }
|