1 /*
2 * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciInlineKlass.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "opto/addnode.hpp"
30 #include "opto/castnode.hpp"
31 #include "opto/graphKit.hpp"
32 #include "opto/inlinetypenode.hpp"
33 #include "opto/rootnode.hpp"
34 #include "opto/phaseX.hpp"
35
36 // Clones the inline type to handle control flow merges involving multiple inline types.
37 // The inputs are replaced by PhiNodes to represent the merged values for the given region.
38 InlineTypeNode* InlineTypeNode::clone_with_phis(PhaseGVN* gvn, Node* region, bool is_init) {
39 InlineTypeNode* vt = clone()->as_InlineType();
40 const Type* t = Type::get_const_type(inline_klass());
41 gvn->set_type(vt, t);
42 vt->as_InlineType()->set_type(t);
43
44 // Create a PhiNode for merging the oop values
45 PhiNode* oop = PhiNode::make(region, vt->get_oop(), t);
46 gvn->set_type(oop, t);
47 gvn->record_for_igvn(oop);
48 vt->set_oop(oop);
49
50 // Create a PhiNode for merging the is_buffered values
51 t = Type::get_const_basic_type(T_BOOLEAN);
52 Node* is_buffered_node = PhiNode::make(region, vt->get_is_buffered(), t);
53 gvn->set_type(is_buffered_node, t);
54 gvn->record_for_igvn(is_buffered_node);
55 vt->set_req(IsBuffered, is_buffered_node);
56
57 // Create a PhiNode for merging the is_init values
58 Node* is_init_node;
59 if (is_init) {
60 is_init_node = gvn->intcon(1);
61 } else {
62 t = Type::get_const_basic_type(T_BOOLEAN);
63 is_init_node = PhiNode::make(region, vt->get_is_init(), t);
64 gvn->set_type(is_init_node, t);
65 gvn->record_for_igvn(is_init_node);
66 }
67 vt->set_req(IsInit, is_init_node);
68
69 // Create a PhiNode each for merging the field values
70 for (uint i = 0; i < vt->field_count(); ++i) {
71 ciType* type = vt->field_type(i);
72 Node* value = vt->field_value(i);
73 // We limit scalarization for inline types with circular fields and can therefore observe nodes
74 // of the same type but with different scalarization depth during IGVN. To avoid inconsistencies
75 // during merging, make sure that we only create Phis for fields that are guaranteed to be scalarized.
76 bool no_circularity = !gvn->C->has_circular_inline_type() || !gvn->is_IterGVN() || field_is_flattened(i);
77 if (value->is_InlineType() && no_circularity) {
78 // Handle inline type fields recursively
79 value = value->as_InlineType()->clone_with_phis(gvn, region);
80 } else {
81 t = Type::get_const_type(type);
82 value = PhiNode::make(region, value, t);
83 gvn->set_type(value, t);
84 gvn->record_for_igvn(value);
85 }
86 vt->set_field_value(i, value);
87 }
88 gvn->record_for_igvn(vt);
89 return vt;
90 }
91
92 // Checks if the inputs of the InlineTypeNode were replaced by PhiNodes
93 // for the given region (see InlineTypeNode::clone_with_phis).
94 bool InlineTypeNode::has_phi_inputs(Node* region) {
95 // Check oop input
96 bool result = get_oop()->is_Phi() && get_oop()->as_Phi()->region() == region;
97 #ifdef ASSERT
98 if (result) {
99 // Check all field value inputs for consistency
100 for (uint i = Values; i < field_count(); ++i) {
101 Node* n = in(i);
102 if (n->is_InlineType()) {
103 assert(n->as_InlineType()->has_phi_inputs(region), "inconsistent phi inputs");
104 } else {
105 assert(n->is_Phi() && n->as_Phi()->region() == region, "inconsistent phi inputs");
106 }
107 }
108 }
109 #endif
110 return result;
111 }
112
113 // Merges 'this' with 'other' by updating the input PhiNodes added by 'clone_with_phis'
114 InlineTypeNode* InlineTypeNode::merge_with(PhaseGVN* gvn, const InlineTypeNode* other, int pnum, bool transform) {
115 // Merge oop inputs
116 PhiNode* phi = get_oop()->as_Phi();
117 phi->set_req(pnum, other->get_oop());
118 if (transform) {
119 set_oop(gvn->transform(phi));
120 }
121
122 // Merge is_buffered inputs
123 phi = get_is_buffered()->as_Phi();
124 phi->set_req(pnum, other->get_is_buffered());
125 if (transform) {
126 set_req(IsBuffered, gvn->transform(phi));
127 }
128
129 // Merge is_init inputs
130 Node* is_init = get_is_init();
131 if (is_init->is_Phi()) {
132 phi = is_init->as_Phi();
133 phi->set_req(pnum, other->get_is_init());
134 if (transform) {
135 set_req(IsInit, gvn->transform(phi));
136 }
137 } else {
138 assert(is_init->find_int_con(0) == 1, "only with a non null inline type");
139 }
140
141 // Merge field values
142 for (uint i = 0; i < field_count(); ++i) {
143 Node* val1 = field_value(i);
144 Node* val2 = other->field_value(i);
145 if (val1->is_InlineType()) {
146 val1->as_InlineType()->merge_with(gvn, val2->as_InlineType(), pnum, transform);
147 } else {
148 assert(val1->is_Phi(), "must be a phi node");
149 val1->set_req(pnum, val2);
150 }
151 if (transform) {
152 set_field_value(i, gvn->transform(val1));
153 }
154 }
155 return this;
156 }
157
158 // Adds a new merge path to an inline type node with phi inputs
159 void InlineTypeNode::add_new_path(Node* region) {
160 assert(has_phi_inputs(region), "must have phi inputs");
161
162 PhiNode* phi = get_oop()->as_Phi();
163 phi->add_req(NULL);
164 assert(phi->req() == region->req(), "must be same size as region");
165
166 phi = get_is_buffered()->as_Phi();
167 phi->add_req(NULL);
168 assert(phi->req() == region->req(), "must be same size as region");
169
170 phi = get_is_init()->as_Phi();
171 phi->add_req(NULL);
172 assert(phi->req() == region->req(), "must be same size as region");
173
174 for (uint i = 0; i < field_count(); ++i) {
175 Node* val = field_value(i);
176 if (val->is_InlineType()) {
177 val->as_InlineType()->add_new_path(region);
178 } else {
179 val->as_Phi()->add_req(NULL);
180 assert(val->req() == region->req(), "must be same size as region");
181 }
182 }
183 }
184
185 Node* InlineTypeNode::field_value(uint index) const {
186 assert(index < field_count(), "index out of bounds");
187 return in(Values + index);
188 }
189
190 // Get the value of the field at the given offset.
191 // If 'recursive' is true, flattened inline type fields will be resolved recursively.
192 Node* InlineTypeNode::field_value_by_offset(int offset, bool recursive) const {
193 // If the field at 'offset' belongs to a flattened inline type field, 'index' refers to the
194 // corresponding InlineTypeNode input and 'sub_offset' is the offset in flattened inline type.
195 int index = inline_klass()->field_index_by_offset(offset);
196 int sub_offset = offset - field_offset(index);
197 Node* value = field_value(index);
198 assert(value != NULL, "field value not found");
199 if (recursive && value->is_InlineType()) {
200 if (field_is_flattened(index)) {
201 // Flattened inline type field
202 InlineTypeNode* vt = value->as_InlineType();
203 sub_offset += vt->inline_klass()->first_field_offset(); // Add header size
204 return vt->field_value_by_offset(sub_offset, recursive);
205 } else {
206 assert(sub_offset == 0, "should not have a sub offset");
207 return value;
208 }
209 }
210 assert(!(recursive && value->is_InlineType()), "should not be an inline type");
211 assert(sub_offset == 0, "offset mismatch");
212 return value;
213 }
214
215 void InlineTypeNode::set_field_value(uint index, Node* value) {
216 assert(index < field_count(), "index out of bounds");
217 set_req(Values + index, value);
218 }
219
220 void InlineTypeNode::set_field_value_by_offset(int offset, Node* value) {
221 set_field_value(field_index(offset), value);
222 }
223
224 int InlineTypeNode::field_offset(uint index) const {
225 assert(index < field_count(), "index out of bounds");
226 return inline_klass()->declared_nonstatic_field_at(index)->offset_in_bytes();
227 }
228
229 uint InlineTypeNode::field_index(int offset) const {
230 uint i = 0;
231 for (; i < field_count() && field_offset(i) != offset; i++) { }
232 assert(i < field_count(), "field not found");
233 return i;
234 }
235
236 ciType* InlineTypeNode::field_type(uint index) const {
237 assert(index < field_count(), "index out of bounds");
238 return inline_klass()->declared_nonstatic_field_at(index)->type();
239 }
240
241 bool InlineTypeNode::field_is_flattened(uint index) const {
242 assert(index < field_count(), "index out of bounds");
243 ciField* field = inline_klass()->declared_nonstatic_field_at(index);
244 assert(!field->is_flattened() || field->type()->is_inlinetype(), "must be an inline type");
245 return field->is_flattened();
246 }
247
248 bool InlineTypeNode::field_is_null_free(uint index) const {
249 assert(index < field_count(), "index out of bounds");
250 ciField* field = inline_klass()->declared_nonstatic_field_at(index);
251 assert(!field->is_flattened() || field->type()->is_inlinetype(), "must be an inline type");
252 return field->is_null_free();
253 }
254
255 void InlineTypeNode::make_scalar_in_safepoint(PhaseIterGVN* igvn, Unique_Node_List& worklist, SafePointNode* sfpt) {
256 ciInlineKlass* vk = inline_klass();
257 uint nfields = vk->nof_nonstatic_fields();
258 JVMState* jvms = sfpt->jvms();
259 // Replace safepoint edge by SafePointScalarObjectNode and add field values
260 assert(jvms != NULL, "missing JVMS");
261 uint first_ind = (sfpt->req() - jvms->scloff());
262 SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(type()->isa_instptr(),
263 #ifdef ASSERT
264 NULL,
265 #endif
266 first_ind, nfields);
267 sobj->init_req(0, igvn->C->root());
268 // Nullable inline types have an IsInit field that needs
269 // to be checked before using the field values.
270 if (!igvn->type(get_is_init())->is_int()->is_con(1)) {
271 sfpt->add_req(get_is_init());
272 } else {
273 sfpt->add_req(igvn->C->top());
274 }
275 // Iterate over the inline type fields in order of increasing
276 // offset and add the field values to the safepoint.
277 for (uint j = 0; j < nfields; ++j) {
278 int offset = vk->nonstatic_field_at(j)->offset_in_bytes();
279 Node* value = field_value_by_offset(offset, true /* include flattened inline type fields */);
280 if (value->is_InlineType()) {
281 // Add inline type field to the worklist to process later
282 worklist.push(value);
283 }
284 sfpt->add_req(value);
285 }
286 jvms->set_endoff(sfpt->req());
287 sobj = igvn->transform(sobj)->as_SafePointScalarObject();
288 igvn->rehash_node_delayed(sfpt);
289 for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) {
290 Node* debug = sfpt->in(i);
291 if (debug != NULL && debug->uncast() == this) {
292 sfpt->set_req(i, sobj);
293 }
294 }
295 }
296
297 void InlineTypeNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop) {
298 // If the inline type has a constant or loaded oop, use the oop instead of scalarization
299 // in the safepoint to avoid keeping field loads live just for the debug info.
300 Node* oop = get_oop();
301 bool use_oop = allow_oop && is_allocated(igvn) &&
302 (oop->is_Con() || oop->is_Parm() || oop->is_Load() || (oop->isa_DecodeN() && oop->in(1)->is_Load()));
303
304 ResourceMark rm;
305 Unique_Node_List safepoints;
306 Unique_Node_List vt_worklist;
307 Unique_Node_List worklist;
308 worklist.push(this);
309 while (worklist.size() > 0) {
310 Node* n = worklist.pop();
311 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
312 Node* use = n->fast_out(i);
313 if (use->is_SafePoint() && !use->is_CallLeaf() && (!use->is_Call() || use->as_Call()->has_debug_use(n))) {
314 safepoints.push(use);
315 } else if (use->is_ConstraintCast()) {
316 worklist.push(use);
317 }
318 }
319 }
320
321 // Process all safepoint uses and scalarize inline type
322 while (safepoints.size() > 0) {
323 SafePointNode* sfpt = safepoints.pop()->as_SafePoint();
324 if (use_oop) {
325 for (uint i = sfpt->jvms()->debug_start(); i < sfpt->jvms()->debug_end(); i++) {
326 Node* debug = sfpt->in(i);
327 if (debug != NULL && debug->uncast() == this) {
328 sfpt->set_req(i, get_oop());
329 }
330 }
331 igvn->rehash_node_delayed(sfpt);
332 } else {
333 make_scalar_in_safepoint(igvn, vt_worklist, sfpt);
334 }
335 }
336 // Now scalarize non-flattened fields
337 for (uint i = 0; i < vt_worklist.size(); ++i) {
338 InlineTypeNode* vt = vt_worklist.at(i)->isa_InlineType();
339 vt->make_scalar_in_safepoints(igvn);
340 }
341 if (outcnt() == 0) {
342 igvn->_worklist.push(this);
343 }
344 }
345
346 const TypePtr* InlineTypeNode::field_adr_type(Node* base, int offset, ciInstanceKlass* holder, DecoratorSet decorators, PhaseGVN& gvn) const {
347 const TypeAryPtr* ary_type = gvn.type(base)->isa_aryptr();
348 const TypePtr* adr_type = NULL;
349 bool is_array = ary_type != NULL;
350 if ((decorators & C2_MISMATCHED) != 0) {
351 adr_type = TypeRawPtr::BOTTOM;
352 } else if (is_array) {
353 // In the case of a flattened inline type array, each field has its own slice
354 adr_type = ary_type->with_field_offset(offset)->add_offset(Type::OffsetBot);
355 } else {
356 ciField* field = holder->get_field_by_offset(offset, false);
357 assert(field != NULL, "field not found");
358 adr_type = gvn.C->alias_type(field)->adr_type();
359 }
360 return adr_type;
361 }
362
363 // We limit scalarization for inline types with circular fields and can therefore observe
364 // nodes of same type but with different scalarization depth during GVN. This method adjusts
365 // the scalarization depth to avoid inconsistencies during merging.
366 InlineTypeNode* InlineTypeNode::adjust_scalarization_depth(GraphKit* kit) {
367 if (!kit->C->has_circular_inline_type()) {
368 return this;
369 }
370 GrowableArray<ciType*> visited;
371 visited.push(inline_klass());
372 return adjust_scalarization_depth_impl(kit, visited);
373 }
374
375 InlineTypeNode* InlineTypeNode::adjust_scalarization_depth_impl(GraphKit* kit, GrowableArray<ciType*>& visited) {
376 InlineTypeNode* val = this;
377 for (uint i = 0; i < field_count(); ++i) {
378 Node* value = field_value(i);
379 Node* new_value = value;
380 ciType* ft = field_type(i);
381 if (value->is_InlineType()) {
382 if (!field_is_flattened(i) && visited.contains(ft)) {
383 new_value = value->as_InlineType()->buffer(kit)->get_oop();
384 } else {
385 int old_len = visited.length();
386 visited.push(ft);
387 new_value = value->as_InlineType()->adjust_scalarization_depth_impl(kit, visited);
388 visited.trunc_to(old_len);
389 }
390 } else if (ft->is_inlinetype() && !visited.contains(ft)) {
391 int old_len = visited.length();
392 visited.push(ft);
393 new_value = make_from_oop_impl(kit, value, ft->as_inline_klass(), field_is_null_free(i), visited);
394 visited.trunc_to(old_len);
395 }
396 if (value != new_value) {
397 if (val == this) {
398 val = clone()->as_InlineType();
399 }
400 val->set_field_value(i, new_value);
401 }
402 }
403 return (val == this) ? this : kit->gvn().transform(val)->as_InlineType();
404 }
405
406 void InlineTypeNode::load(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, GrowableArray<ciType*>& visited, int holder_offset, DecoratorSet decorators) {
407 // Initialize the inline type by loading its field values from
408 // memory and adding the values as input edges to the node.
409 for (uint i = 0; i < field_count(); ++i) {
410 int offset = holder_offset + field_offset(i);
411 Node* value = NULL;
412 ciType* ft = field_type(i);
413 bool null_free = field_is_null_free(i);
414 if (null_free && ft->as_inline_klass()->is_empty()) {
415 // Loading from a field of an empty inline type. Just return the default instance.
416 value = make_default_impl(kit->gvn(), ft->as_inline_klass(), visited);
417 } else if (field_is_flattened(i)) {
418 // Recursively load the flattened inline type field
419 value = make_from_flattened_impl(kit, ft->as_inline_klass(), base, ptr, holder, offset, decorators, visited);
420 } else {
421 const TypeOopPtr* oop_ptr = kit->gvn().type(base)->isa_oopptr();
422 bool is_array = (oop_ptr->isa_aryptr() != NULL);
423 bool mismatched = (decorators & C2_MISMATCHED) != 0;
424 if (base->is_Con() && !is_array && !mismatched) {
425 // If the oop to the inline type is constant (static final field), we can
426 // also treat the fields as constants because the inline type is immutable.
427 ciObject* constant_oop = oop_ptr->const_oop();
428 ciField* field = holder->get_field_by_offset(offset, false);
429 assert(field != NULL, "field not found");
430 ciConstant constant = constant_oop->as_instance()->field_value(field);
431 const Type* con_type = Type::make_from_constant(constant, /*require_const=*/ true);
432 assert(con_type != NULL, "type not found");
433 value = kit->gvn().transform(kit->makecon(con_type));
434 // Check type of constant which might be more precise than the static field type
435 if (con_type->is_inlinetypeptr() && !con_type->is_zero_type()) {
436 ft = con_type->inline_klass();
437 null_free = true;
438 }
439 } else {
440 // Load field value from memory
441 const TypePtr* adr_type = field_adr_type(base, offset, holder, decorators, kit->gvn());
442 Node* adr = kit->basic_plus_adr(base, ptr, offset);
443 BasicType bt = type2field[ft->basic_type()];
444 assert(is_java_primitive(bt) || adr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
445 const Type* val_type = Type::get_const_type(ft);
446 value = kit->access_load_at(base, adr, adr_type, val_type, bt, is_array ? (decorators | IS_ARRAY) : decorators);
447 }
448 // Loading a non-flattened inline type from memory
449 if (visited.contains(ft)) {
450 kit->C->set_has_circular_inline_type(true);
451 } else if (ft->is_inlinetype()) {
452 int old_len = visited.length();
453 visited.push(ft);
454 value = make_from_oop_impl(kit, value, ft->as_inline_klass(), null_free, visited);
455 visited.trunc_to(old_len);
456 }
457 }
458 set_field_value(i, value);
459 }
460 }
461
462 void InlineTypeNode::store_flattened(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators) const {
463 if (kit->gvn().type(base)->isa_aryptr()) {
464 kit->C->set_flattened_accesses();
465 }
466 // The inline type is embedded into the object without an oop header. Subtract the
467 // offset of the first field to account for the missing header when storing the values.
468 if (holder == NULL) {
469 holder = inline_klass();
470 }
471 holder_offset -= inline_klass()->first_field_offset();
472 store(kit, base, ptr, holder, holder_offset, decorators);
473 }
474
475 void InlineTypeNode::store(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators) const {
476 // Write field values to memory
477 for (uint i = 0; i < field_count(); ++i) {
478 int offset = holder_offset + field_offset(i);
479 Node* value = field_value(i);
480 ciType* ft = field_type(i);
481 if (field_is_flattened(i)) {
482 // Recursively store the flattened inline type field
483 value->as_InlineType()->store_flattened(kit, base, ptr, holder, offset, decorators);
484 } else {
485 // Store field value to memory
486 const TypePtr* adr_type = field_adr_type(base, offset, holder, decorators, kit->gvn());
487 Node* adr = kit->basic_plus_adr(base, ptr, offset);
488 BasicType bt = type2field[ft->basic_type()];
489 assert(is_java_primitive(bt) || adr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
490 const Type* val_type = Type::get_const_type(ft);
491 bool is_array = (kit->gvn().type(base)->isa_aryptr() != NULL);
492 kit->access_store_at(base, adr, adr_type, value, val_type, bt, is_array ? (decorators | IS_ARRAY) : decorators);
493 }
494 }
495 }
496
497 InlineTypeNode* InlineTypeNode::buffer(GraphKit* kit, bool safe_for_replace) {
498 if (kit->gvn().find_int_con(get_is_buffered(), 0) == 1) {
499 // Already buffered
500 return this;
501 }
502
503 // Check if inline type is already buffered
504 Node* not_buffered_ctl = kit->top();
505 Node* not_null_oop = kit->null_check_oop(get_oop(), ¬_buffered_ctl, /* never_see_null = */ false, safe_for_replace);
506 if (not_buffered_ctl->is_top()) {
507 // Already buffered
508 InlineTypeNode* vt = clone()->as_InlineType();
509 vt->set_is_buffered(kit->gvn());
510 vt = kit->gvn().transform(vt)->as_InlineType();
511 if (safe_for_replace) {
512 kit->replace_in_map(this, vt);
513 }
514 return vt;
515 }
516 Node* buffered_ctl = kit->control();
517 kit->set_control(not_buffered_ctl);
518
519 // Inline type is not buffered, check if it is null.
520 Node* null_ctl = kit->top();
521 kit->null_check_common(get_is_init(), T_INT, false, &null_ctl);
522 bool null_free = null_ctl->is_top();
523
524 RegionNode* region = new RegionNode(4);
525 PhiNode* oop = PhiNode::make(region, not_null_oop, type()->join_speculative(null_free ? TypePtr::NOTNULL : TypePtr::BOTTOM));
526
527 // InlineType is already buffered
528 region->init_req(1, buffered_ctl);
529 oop->init_req(1, not_null_oop);
530
531 // InlineType is null
532 region->init_req(2, null_ctl);
533 oop->init_req(2, kit->gvn().zerocon(T_OBJECT));
534
535 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO);
536 PhiNode* mem = PhiNode::make(region, kit->merged_memory(), Type::MEMORY, TypePtr::BOTTOM);
537
538 int bci = kit->bci();
539 bool reexecute = kit->jvms()->should_reexecute();
540 if (!kit->stopped()) {
541 assert(!is_allocated(&kit->gvn()), "already buffered");
542
543 // Allocate and initialize buffer
544 PreserveJVMState pjvms(kit);
545 // Propagate re-execution state and bci
546 kit->set_bci(bci);
547 kit->jvms()->set_bci(bci);
548 kit->jvms()->set_should_reexecute(reexecute);
549
550 kit->kill_dead_locals();
551 ciInlineKlass* vk = inline_klass();
552 Node* klass_node = kit->makecon(TypeKlassPtr::make(vk));
553 Node* alloc_oop = kit->new_instance(klass_node, NULL, NULL, /* deoptimize_on_exception */ true, this);
554 store(kit, alloc_oop, alloc_oop, vk);
555
556 // Do not let stores that initialize this buffer be reordered with a subsequent
557 // store that would make this buffer accessible by other threads.
558 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_oop, &kit->gvn());
559 assert(alloc != NULL, "must have an allocation node");
560 kit->insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress));
561
562 region->init_req(3, kit->control());
563 oop ->init_req(3, alloc_oop);
564 io ->init_req(3, kit->i_o());
565 mem ->init_req(3, kit->merged_memory());
566 }
567
568 // Update GraphKit
569 kit->set_control(kit->gvn().transform(region));
570 kit->set_i_o(kit->gvn().transform(io));
571 kit->set_all_memory(kit->gvn().transform(mem));
572 kit->record_for_igvn(region);
573 kit->record_for_igvn(oop);
574 kit->record_for_igvn(io);
575 kit->record_for_igvn(mem);
576
577 // Use cloned InlineTypeNode to propagate oop from now on
578 Node* res_oop = kit->gvn().transform(oop);
579 InlineTypeNode* vt = clone()->as_InlineType();
580 vt->set_oop(res_oop);
581 vt->set_is_buffered(kit->gvn());
582 vt = kit->gvn().transform(vt)->as_InlineType();
583 if (safe_for_replace) {
584 kit->replace_in_map(this, vt);
585 }
586 // InlineTypeNode::remove_redundant_allocations piggybacks on split if.
587 // Make sure it gets a chance to remove this allocation.
588 kit->C->set_has_split_ifs(true);
589 return vt;
590 }
591
592 bool InlineTypeNode::is_allocated(PhaseGVN* phase) const {
593 if (phase->find_int_con(get_is_buffered(), 0) == 1) {
594 return true;
595 }
596 Node* oop = get_oop();
597 const Type* oop_type = (phase != NULL) ? phase->type(oop) : oop->bottom_type();
598 return !oop_type->maybe_null();
599 }
600
601 // When a call returns multiple values, it has several result
602 // projections, one per field. Replacing the result of the call by an
603 // inline type node (after late inlining) requires that for each result
604 // projection, we find the corresponding inline type field.
605 void InlineTypeNode::replace_call_results(GraphKit* kit, CallNode* call, Compile* C, bool null_free) {
606 ciInlineKlass* vk = inline_klass();
607 for (DUIterator_Fast imax, i = call->fast_outs(imax); i < imax; i++) {
608 ProjNode* pn = call->fast_out(i)->as_Proj();
609 uint con = pn->_con;
610 Node* field = NULL;
611 if (con == TypeFunc::Parms) {
612 field = get_oop();
613 } else if (!null_free && con == (call->tf()->range_cc()->cnt() - 1)) {
614 field = get_is_init();
615 } else if (con > TypeFunc::Parms) {
616 uint field_nb = con - (TypeFunc::Parms+1);
617 int extra = 0;
618 for (uint j = 0; j < field_nb - extra; j++) {
619 ciField* f = vk->nonstatic_field_at(j);
620 BasicType bt = f->type()->basic_type();
621 if (bt == T_LONG || bt == T_DOUBLE) {
622 extra++;
623 }
624 }
625 ciField* f = vk->nonstatic_field_at(field_nb - extra);
626 field = field_value_by_offset(f->offset_in_bytes(), true);
627 }
628 if (field != NULL) {
629 C->gvn_replace_by(pn, field);
630 C->initial_gvn()->hash_delete(pn);
631 pn->set_req(0, C->top());
632 --i; --imax;
633 }
634 }
635 }
636
637 Node* InlineTypeNode::allocate_fields(GraphKit* kit) {
638 InlineTypeNode* vt = clone()->as_InlineType();
639 for (uint i = 0; i < field_count(); i++) {
640 Node* value = field_value(i);
641 if (field_is_flattened(i)) {
642 // Flattened inline type field
643 vt->set_field_value(i, value->as_InlineType()->allocate_fields(kit));
644 } else if (value->is_InlineType()) {
645 // Non-flattened inline type field
646 vt->set_field_value(i, value->as_InlineType()->buffer(kit));
647 }
648 }
649 vt = kit->gvn().transform(vt)->as_InlineType();
650 kit->replace_in_map(this, vt);
651 return vt;
652 }
653
654 // Replace a buffer allocation by a dominating allocation
655 static void replace_allocation(PhaseIterGVN* igvn, Node* res, Node* dom) {
656 // Remove initializing stores and GC barriers
657 for (DUIterator_Fast imax, i = res->fast_outs(imax); i < imax; i++) {
658 Node* use = res->fast_out(i);
659 if (use->is_AddP()) {
660 for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) {
661 Node* store = use->fast_out(j)->isa_Store();
662 if (store != NULL) {
663 igvn->rehash_node_delayed(store);
664 igvn->replace_in_uses(store, store->in(MemNode::Memory));
665 }
666 }
667 } else if (use->Opcode() == Op_CastP2X) {
668 if (UseG1GC && use->find_out_with(Op_XorX)->in(1) != use) {
669 // The G1 pre-barrier uses a CastP2X both for the pointer of the object
670 // we store into, as well as the value we are storing. Skip if this is a
671 // barrier for storing 'res' into another object.
672 continue;
673 }
674 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
675 bs->eliminate_gc_barrier(igvn, use);
676 --i; --imax;
677 }
678 }
679 igvn->replace_node(res, dom);
680 }
681
682 Node* InlineTypeNode::Ideal(PhaseGVN* phase, bool can_reshape) {
683 Node* oop = get_oop();
684 if (!is_larval(phase) &&
685 is_default(phase) &&
686 inline_klass()->is_initialized() &&
687 (!oop->is_Con() || phase->type(oop)->is_zero_type())) {
688 // Use the pre-allocated oop for default inline types
689 set_oop(default_oop(*phase, inline_klass()));
690 assert(is_allocated(phase), "should now be allocated");
691 return this;
692 }
693 if (oop->isa_InlineType() && !phase->type(oop)->maybe_null()) {
694 InlineTypeNode* vtptr = oop->as_InlineType();
695 set_oop(vtptr->get_oop());
696 set_is_buffered(*phase);
697 set_is_init(*phase);
698 for (uint i = Values; i < vtptr->req(); ++i) {
699 set_req(i, vtptr->in(i));
700 }
701 return this;
702 }
703 if (!is_allocated(phase)) {
704 // Save base oop if fields are loaded from memory and the inline
705 // type is not buffered (in this case we should not use the oop).
706 Node* base = is_loaded(phase);
707 if (base != NULL && !phase->type(base)->maybe_null()) {
708 set_oop(base);
709 assert(is_allocated(phase), "should now be allocated");
710 return this;
711 }
712 }
713
714 if (can_reshape) {
715 PhaseIterGVN* igvn = phase->is_IterGVN();
716 if (is_allocated(phase)) {
717 // Search for and remove re-allocations of this inline type. Ignore scalar replaceable ones,
718 // they will be removed anyway and changing the memory chain will confuse other optimizations.
719 // This can happen with late inlining when we first allocate an inline type argument
720 // but later decide to inline the call after the callee code also triggered allocation.
721 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
722 AllocateNode* alloc = fast_out(i)->isa_Allocate();
723 if (alloc != NULL && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable) {
724 // Found a re-allocation
725 Node* res = alloc->result_cast();
726 if (res != NULL && res->is_CheckCastPP()) {
727 // Replace allocation by oop and unlink AllocateNode
728 replace_allocation(igvn, res, oop);
729 igvn->replace_input_of(alloc, AllocateNode::InlineType, igvn->C->top());
730 --i; --imax;
731 }
732 }
733 }
734 }
735 }
736
737 return NULL;
738 }
739
740 InlineTypeNode* InlineTypeNode::make_uninitialized(PhaseGVN& gvn, ciInlineKlass* vk, bool null_free) {
741 // Create a new InlineTypeNode with uninitialized values and NULL oop
742 Node* oop = (vk->is_empty() && vk->is_initialized()) ? default_oop(gvn, vk) : gvn.zerocon(T_PRIMITIVE_OBJECT);
743 InlineTypeNode* vt = new InlineTypeNode(vk, oop, null_free);
744 vt->set_is_buffered(gvn, vk->is_empty() && vk->is_initialized());
745 vt->set_is_init(gvn);
746 return vt;
747 }
748
749 Node* InlineTypeNode::default_oop(PhaseGVN& gvn, ciInlineKlass* vk) {
750 // Returns the constant oop of the default inline type allocation
751 return gvn.makecon(TypeInstPtr::make(vk->default_instance()));
752 }
753
754 InlineTypeNode* InlineTypeNode::make_default(PhaseGVN& gvn, ciInlineKlass* vk) {
755 GrowableArray<ciType*> visited;
756 visited.push(vk);
757 return make_default_impl(gvn, vk, visited);
758 }
759
760 InlineTypeNode* InlineTypeNode::make_default_impl(PhaseGVN& gvn, ciInlineKlass* vk, GrowableArray<ciType*>& visited) {
761 // Create a new InlineTypeNode with default values
762 Node* oop = vk->is_initialized() ? default_oop(gvn, vk) : gvn.zerocon(T_PRIMITIVE_OBJECT);
763 InlineTypeNode* vt = new InlineTypeNode(vk, oop, /* null_free= */ true);
764 vt->set_is_buffered(gvn, vk->is_initialized());
765 vt->set_is_init(gvn);
766 for (uint i = 0; i < vt->field_count(); ++i) {
767 ciType* ft = vt->field_type(i);
768 Node* value = gvn.zerocon(ft->basic_type());
769 if (!vt->field_is_flattened(i) && visited.contains(ft)) {
770 gvn.C->set_has_circular_inline_type(true);
771 } else if (ft->is_inlinetype()) {
772 int old_len = visited.length();
773 visited.push(ft);
774 ciInlineKlass* vk = ft->as_inline_klass();
775 if (vt->field_is_null_free(i)) {
776 value = make_default_impl(gvn, vk, visited);
777 } else {
778 value = make_null_impl(gvn, vk, visited);
779 }
780 visited.trunc_to(old_len);
781 }
782 vt->set_field_value(i, value);
783 }
784 vt = gvn.transform(vt)->as_InlineType();
785 assert(vt->is_default(&gvn), "must be the default inline type");
786 return vt;
787 }
788
789 bool InlineTypeNode::is_default(PhaseGVN* gvn) const {
790 const Type* tinit = gvn->type(in(IsInit));
791 if (!tinit->isa_int() || !tinit->is_int()->is_con(1)) {
792 return false; // May be null
793 }
794 for (uint i = 0; i < field_count(); ++i) {
795 ciType* ft = field_type(i);
796 Node* value = field_value(i);
797 if (field_is_null_free(i)) {
798 if (!value->is_InlineType() || !value->as_InlineType()->is_default(gvn)) {
799 return false;
800 }
801 continue;
802 } else if (value->is_InlineType()) {
803 value = value->as_InlineType()->get_oop();
804 }
805 if (!gvn->type(value)->is_zero_type()) {
806 return false;
807 }
808 }
809 return true;
810 }
811
812 InlineTypeNode* InlineTypeNode::make_from_oop(GraphKit* kit, Node* oop, ciInlineKlass* vk, bool null_free) {
813 GrowableArray<ciType*> visited;
814 visited.push(vk);
815 return make_from_oop_impl(kit, oop, vk, null_free, visited);
816 }
817
818 InlineTypeNode* InlineTypeNode::make_from_oop_impl(GraphKit* kit, Node* oop, ciInlineKlass* vk, bool null_free, GrowableArray<ciType*>& visited) {
819 PhaseGVN& gvn = kit->gvn();
820
821 if (vk->is_empty() && null_free) {
822 InlineTypeNode* def = make_default_impl(gvn, vk, visited);
823 kit->record_for_igvn(def);
824 return def;
825 }
826 // Create and initialize an InlineTypeNode by loading all field
827 // values from a heap-allocated version and also save the oop.
828 InlineTypeNode* vt = NULL;
829
830 if (oop->isa_InlineType()) {
831 return oop->as_InlineType();
832 } else if (gvn.type(oop)->maybe_null()) {
833 // Add a null check because the oop may be null
834 Node* null_ctl = kit->top();
835 Node* not_null_oop = kit->null_check_oop(oop, &null_ctl);
836 if (kit->stopped()) {
837 // Constant null
838 kit->set_control(null_ctl);
839 if (null_free) {
840 vt = make_default_impl(gvn, vk, visited);
841 } else {
842 vt = make_null_impl(gvn, vk, visited);
843 }
844 kit->record_for_igvn(vt);
845 return vt;
846 }
847 vt = new InlineTypeNode(vk, not_null_oop, null_free);
848 vt->set_is_buffered(gvn);
849 vt->set_is_init(gvn);
850 vt->load(kit, not_null_oop, not_null_oop, vk, visited);
851
852 if (null_ctl != kit->top()) {
853 InlineTypeNode* null_vt = NULL;
854 if (null_free) {
855 null_vt = make_default_impl(gvn, vk, visited);
856 } else {
857 null_vt = make_null_impl(gvn, vk, visited);
858 }
859 Node* region = new RegionNode(3);
860 region->init_req(1, kit->control());
861 region->init_req(2, null_ctl);
862
863 vt = vt->clone_with_phis(&gvn, region);
864 vt->merge_with(&gvn, null_vt, 2, true);
865 if (!null_free) {
866 vt->set_oop(oop);
867 }
868 kit->set_control(gvn.transform(region));
869 }
870 } else {
871 // Oop can never be null
872 vt = new InlineTypeNode(vk, oop, /* null_free= */ true);
873 Node* init_ctl = kit->control();
874 vt->set_is_buffered(gvn);
875 vt->set_is_init(gvn);
876 vt->load(kit, oop, oop, vk, visited);
877 // TODO 8284443
878 // assert(!null_free || vt->as_InlineType()->is_default(&gvn) || init_ctl != kit->control() || !gvn.type(oop)->is_inlinetypeptr() || oop->is_Con() || oop->Opcode() == Op_InlineType ||
879 // AllocateNode::Ideal_allocation(oop, &gvn) != NULL || vt->as_InlineType()->is_loaded(&gvn) == oop, "inline type should be loaded");
880 }
881 assert(!null_free || vt->is_allocated(&gvn), "inline type should be allocated");
882 kit->record_for_igvn(vt);
883 return gvn.transform(vt)->as_InlineType();
884 }
885
886 InlineTypeNode* InlineTypeNode::make_from_flattened(GraphKit* kit, ciInlineKlass* vk, Node* obj, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators) {
887 GrowableArray<ciType*> visited;
888 visited.push(vk);
889 return make_from_flattened_impl(kit, vk, obj, ptr, holder, holder_offset, decorators, visited);
890 }
891
892 // GraphKit wrapper for the 'make_from_flattened' method
893 InlineTypeNode* InlineTypeNode::make_from_flattened_impl(GraphKit* kit, ciInlineKlass* vk, Node* obj, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators, GrowableArray<ciType*>& visited) {
894 if (kit->gvn().type(obj)->isa_aryptr()) {
895 kit->C->set_flattened_accesses();
896 }
897 // Create and initialize an InlineTypeNode by loading all field values from
898 // a flattened inline type field at 'holder_offset' or from an inline type array.
899 InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk);
900 // The inline type is flattened into the object without an oop header. Subtract the
901 // offset of the first field to account for the missing header when loading the values.
902 holder_offset -= vk->first_field_offset();
903 vt->load(kit, obj, ptr, holder, visited, holder_offset, decorators);
904 assert(vt->is_loaded(&kit->gvn()) != obj, "holder oop should not be used as flattened inline type oop");
905 return kit->gvn().transform(vt)->as_InlineType();
906 }
907
908 InlineTypeNode* InlineTypeNode::make_from_multi(GraphKit* kit, MultiNode* multi, ciInlineKlass* vk, uint& base_input, bool in, bool null_free) {
909 InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk, null_free);
910 if (!in) {
911 // Keep track of the oop. The returned inline type might already be buffered.
912 Node* oop = kit->gvn().transform(new ProjNode(multi, base_input++));
913 vt->set_oop(oop);
914 }
915 GrowableArray<ciType*> visited;
916 visited.push(vk);
917 vt->initialize_fields(kit, multi, base_input, in, null_free, NULL, visited);
918 return kit->gvn().transform(vt)->as_InlineType();
919 }
920
921 InlineTypeNode* InlineTypeNode::make_larval(GraphKit* kit, bool allocate) const {
922 ciInlineKlass* vk = inline_klass();
923 InlineTypeNode* res = make_uninitialized(kit->gvn(), vk);
924 for (uint i = 1; i < req(); ++i) {
925 res->set_req(i, in(i));
926 }
927
928 if (allocate) {
929 // Re-execute if buffering triggers deoptimization
930 PreserveReexecuteState preexecs(kit);
931 kit->jvms()->set_should_reexecute(true);
932 Node* klass_node = kit->makecon(TypeKlassPtr::make(vk));
933 Node* alloc_oop = kit->new_instance(klass_node, NULL, NULL, true);
934 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_oop, &kit->gvn());
935 alloc->_larval = true;
936
937 store(kit, alloc_oop, alloc_oop, vk);
938 res->set_oop(alloc_oop);
939 }
940 // TODO 8239003
941 //res->set_type(TypeInlineType::make(vk, true));
942 res = kit->gvn().transform(res)->as_InlineType();
943 assert(!allocate || res->is_allocated(&kit->gvn()), "must be allocated");
944 return res;
945 }
946
947 InlineTypeNode* InlineTypeNode::finish_larval(GraphKit* kit) const {
948 Node* obj = get_oop();
949 Node* mark_addr = kit->basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
950 Node* mark = kit->make_load(NULL, mark_addr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered);
951 mark = kit->gvn().transform(new AndXNode(mark, kit->MakeConX(~markWord::larval_bit_in_place)));
952 kit->store_to_memory(kit->control(), mark_addr, mark, TypeX_X->basic_type(), kit->gvn().type(mark_addr)->is_ptr(), MemNode::unordered);
953
954 // Do not let stores that initialize this buffer be reordered with a subsequent
955 // store that would make this buffer accessible by other threads.
956 AllocateNode* alloc = AllocateNode::Ideal_allocation(obj, &kit->gvn());
957 assert(alloc != NULL, "must have an allocation node");
958 kit->insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress));
959
960 ciInlineKlass* vk = inline_klass();
961 InlineTypeNode* res = make_uninitialized(kit->gvn(), vk);
962 for (uint i = 1; i < req(); ++i) {
963 res->set_req(i, in(i));
964 }
965 // TODO 8239003
966 //res->set_type(TypeInlineType::make(vk, false));
967 res = kit->gvn().transform(res)->as_InlineType();
968 return res;
969 }
970
971 bool InlineTypeNode::is_larval(PhaseGVN* gvn) const {
972 if (!is_allocated(gvn)) {
973 return false;
974 }
975
976 Node* oop = get_oop();
977 AllocateNode* alloc = AllocateNode::Ideal_allocation(oop, gvn);
978 return alloc != NULL && alloc->_larval;
979 }
980
981 Node* InlineTypeNode::is_loaded(PhaseGVN* phase, ciInlineKlass* vk, Node* base, int holder_offset) {
982 if (vk == NULL) {
983 vk = inline_klass();
984 }
985 if (field_count() == 0 && vk->is_initialized()) {
986 const Type* tinit = phase->type(in(IsInit));
987 if (tinit->isa_int() && tinit->is_int()->is_con(1)) {
988 assert(is_allocated(phase), "must be allocated");
989 return get_oop();
990 } else {
991 // TODO 8284443
992 return NULL;
993 }
994 }
995 for (uint i = 0; i < field_count(); ++i) {
996 int offset = holder_offset + field_offset(i);
997 Node* value = field_value(i);
998 if (value->is_InlineType()) {
999 InlineTypeNode* vt = value->as_InlineType();
1000 if (vt->type()->inline_klass()->is_empty()) {
1001 continue;
1002 } else if (field_is_flattened(i) && vt->is_InlineType()) {
1003 // Check inline type field load recursively
1004 base = vt->as_InlineType()->is_loaded(phase, vk, base, offset - vt->type()->inline_klass()->first_field_offset());
1005 if (base == NULL) {
1006 return NULL;
1007 }
1008 continue;
1009 } else {
1010 value = vt->get_oop();
1011 if (value->Opcode() == Op_CastPP) {
1012 // Skip CastPP
1013 value = value->in(1);
1014 }
1015 }
1016 }
1017 if (value->isa_DecodeN()) {
1018 // Skip DecodeN
1019 value = value->in(1);
1020 }
1021 if (value->isa_Load()) {
1022 // Check if base and offset of field load matches inline type layout
1023 intptr_t loffset = 0;
1024 Node* lbase = AddPNode::Ideal_base_and_offset(value->in(MemNode::Address), phase, loffset);
1025 if (lbase == NULL || (lbase != base && base != NULL) || loffset != offset) {
1026 return NULL;
1027 } else if (base == NULL) {
1028 // Set base and check if pointer type matches
1029 base = lbase;
1030 const TypeInstPtr* vtptr = phase->type(base)->isa_instptr();
1031 if (vtptr == NULL || !vtptr->instance_klass()->equals(vk)) {
1032 return NULL;
1033 }
1034 }
1035 } else {
1036 return NULL;
1037 }
1038 }
1039 return base;
1040 }
1041
1042 Node* InlineTypeNode::tagged_klass(ciInlineKlass* vk, PhaseGVN& gvn) {
1043 const TypeKlassPtr* tk = TypeKlassPtr::make(vk);
1044 intptr_t bits = tk->get_con();
1045 set_nth_bit(bits, 0);
1046 return gvn.longcon((jlong)bits);
1047 }
1048
1049 void InlineTypeNode::pass_fields(GraphKit* kit, Node* n, uint& base_input, bool in, bool null_free) {
1050 if (!null_free && in) {
1051 n->init_req(base_input++, get_is_init());
1052 }
1053 for (uint i = 0; i < field_count(); i++) {
1054 Node* arg = field_value(i);
1055 if (field_is_flattened(i)) {
1056 // Flattened inline type field
1057 arg->as_InlineType()->pass_fields(kit, n, base_input, in);
1058 } else {
1059 if (arg->is_InlineType()) {
1060 // Non-flattened inline type field
1061 InlineTypeNode* vt = arg->as_InlineType();
1062 assert(n->Opcode() != Op_Return || vt->is_allocated(&kit->gvn()), "inline type field should be allocated on return");
1063 arg = vt->buffer(kit);
1064 }
1065 // Initialize call/return arguments
1066 n->init_req(base_input++, arg);
1067 if (field_type(i)->size() == 2) {
1068 n->init_req(base_input++, kit->top());
1069 }
1070 }
1071 }
1072 // The last argument is used to pass IsInit information to compiled code and not required here.
1073 if (!null_free && !in) {
1074 n->init_req(base_input++, kit->top());
1075 }
1076 }
1077
1078 void InlineTypeNode::initialize_fields(GraphKit* kit, MultiNode* multi, uint& base_input, bool in, bool null_free, Node* null_check_region, GrowableArray<ciType*>& visited) {
1079 PhaseGVN& gvn = kit->gvn();
1080 Node* is_init = NULL;
1081 if (!null_free) {
1082 // Nullable inline type
1083 if (in) {
1084 // Set IsInit field
1085 if (multi->is_Start()) {
1086 is_init = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1087 } else {
1088 is_init = multi->as_Call()->in(base_input);
1089 }
1090 set_req(IsInit, is_init);
1091 base_input++;
1092 }
1093 // Add a null check to make subsequent loads dependent on
1094 assert(null_check_region == NULL, "already set");
1095 if (is_init == NULL) {
1096 // Will only be initialized below, use dummy node for now
1097 is_init = new Node(1);
1098 gvn.set_type_bottom(is_init);
1099 }
1100 Node* null_ctrl = kit->top();
1101 kit->null_check_common(is_init, T_INT, false, &null_ctrl);
1102 Node* non_null_ctrl = kit->control();
1103 null_check_region = new RegionNode(3);
1104 null_check_region->init_req(1, non_null_ctrl);
1105 null_check_region->init_req(2, null_ctrl);
1106 null_check_region = gvn.transform(null_check_region);
1107 kit->set_control(null_check_region);
1108 }
1109
1110 for (uint i = 0; i < field_count(); ++i) {
1111 ciType* type = field_type(i);
1112 Node* parm = NULL;
1113 if (field_is_flattened(i)) {
1114 // Flattened inline type field
1115 InlineTypeNode* vt = make_uninitialized(gvn, type->as_inline_klass());
1116 vt->initialize_fields(kit, multi, base_input, in, true, null_check_region, visited);
1117 parm = gvn.transform(vt);
1118 } else {
1119 if (multi->is_Start()) {
1120 assert(in, "return from start?");
1121 parm = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1122 } else if (in) {
1123 parm = multi->as_Call()->in(base_input);
1124 } else {
1125 parm = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1126 }
1127 // Non-flattened inline type field
1128 if (type->is_inlinetype()) {
1129 if (null_check_region != NULL) {
1130 if (parm->is_InlineType() && kit->C->has_circular_inline_type()) {
1131 parm = parm->as_InlineType()->get_oop();
1132 }
1133 // Holder is nullable, set field to NULL if holder is NULL to avoid loading from uninitialized memory
1134 parm = PhiNode::make(null_check_region, parm, TypeInstPtr::make(TypePtr::BotPTR, type->as_inline_klass()));
1135 parm->set_req(2, kit->zerocon(T_OBJECT));
1136 parm = gvn.transform(parm);
1137 }
1138 if (visited.contains(type)) {
1139 kit->C->set_has_circular_inline_type(true);
1140 } else if (!parm->is_InlineType()) {
1141 int old_len = visited.length();
1142 visited.push(type);
1143 parm = make_from_oop_impl(kit, parm, type->as_inline_klass(), field_is_null_free(i), visited);
1144 visited.trunc_to(old_len);
1145 }
1146 }
1147 base_input += type->size();
1148 }
1149 assert(parm != NULL, "should never be null");
1150 assert(field_value(i) == NULL, "already set");
1151 set_field_value(i, parm);
1152 gvn.record_for_igvn(parm);
1153 }
1154 // The last argument is used to pass IsInit information to compiled code
1155 if (!null_free && !in) {
1156 Node* cmp = is_init->raw_out(0);
1157 is_init = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1158 set_req(IsInit, is_init);
1159 gvn.hash_delete(cmp);
1160 cmp->set_req(1, is_init);
1161 gvn.hash_find_insert(cmp);
1162 base_input++;
1163 }
1164 }
1165
1166 // Search for multiple allocations of this inline type and try to replace them by dominating allocations.
1167 // Equivalent InlineTypeNodes are merged by GVN, so we just need to search for AllocateNode users to find redundant allocations.
1168 void InlineTypeNode::remove_redundant_allocations(PhaseIdealLoop* phase) {
1169 PhaseIterGVN* igvn = &phase->igvn();
1170 // Search for allocations of this inline type. Ignore scalar replaceable ones, they
1171 // will be removed anyway and changing the memory chain will confuse other optimizations.
1172 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1173 AllocateNode* alloc = fast_out(i)->isa_Allocate();
1174 if (alloc != NULL && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable) {
1175 Node* res = alloc->result_cast();
1176 if (res == NULL || !res->is_CheckCastPP()) {
1177 break; // No unique CheckCastPP
1178 }
1179 assert((!is_default(igvn) || !inline_klass()->is_initialized()) && !is_allocated(igvn), "re-allocation should be removed by Ideal transformation");
1180 // Search for a dominating allocation of the same inline type
1181 Node* res_dom = res;
1182 for (DUIterator_Fast jmax, j = fast_outs(jmax); j < jmax; j++) {
1183 AllocateNode* alloc_other = fast_out(j)->isa_Allocate();
1184 if (alloc_other != NULL && alloc_other->in(AllocateNode::InlineType) == this && !alloc_other->_is_scalar_replaceable) {
1185 Node* res_other = alloc_other->result_cast();
1186 if (res_other != NULL && res_other->is_CheckCastPP() && res_other != res_dom &&
1187 phase->is_dominator(res_other->in(0), res_dom->in(0))) {
1188 res_dom = res_other;
1189 }
1190 }
1191 }
1192 if (res_dom != res) {
1193 // Replace allocation by dominating one.
1194 replace_allocation(igvn, res, res_dom);
1195 // The result of the dominated allocation is now unused and will be removed
1196 // later in PhaseMacroExpand::eliminate_allocate_node to not confuse loop opts.
1197 igvn->_worklist.push(alloc);
1198 }
1199 }
1200 }
1201 }
1202
1203 InlineTypeNode* InlineTypeNode::make_null(PhaseGVN& gvn, ciInlineKlass* vk) {
1204 GrowableArray<ciType*> visited;
1205 visited.push(vk);
1206 return make_null_impl(gvn, vk, visited);
1207 }
1208
1209 InlineTypeNode* InlineTypeNode::make_null_impl(PhaseGVN& gvn, ciInlineKlass* vk, GrowableArray<ciType*>& visited) {
1210 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT), /* null_free= */ false);
1211 vt->set_is_buffered(gvn);
1212 vt->set_is_init(gvn, false);
1213 for (uint i = 0; i < vt->field_count(); i++) {
1214 ciType* ft = vt->field_type(i);
1215 Node* value = gvn.zerocon(ft->basic_type());
1216 if (!vt->field_is_flattened(i) && visited.contains(ft)) {
1217 gvn.C->set_has_circular_inline_type(true);
1218 } else if (ft->is_inlinetype()) {
1219 int old_len = visited.length();
1220 visited.push(ft);
1221 value = make_null_impl(gvn, ft->as_inline_klass(), visited);
1222 visited.trunc_to(old_len);
1223 }
1224 vt->set_field_value(i, value);
1225 }
1226 return gvn.transform(vt)->as_InlineType();
1227 }
1228
1229 Node* InlineTypeNode::Identity(PhaseGVN* phase) {
1230 if (get_oop()->is_InlineType()) {
1231 return get_oop();
1232 }
1233 return this;
1234 }
1235
1236 const Type* InlineTypeNode::Value(PhaseGVN* phase) const {
1237 Node* oop = get_oop();
1238 const Type* toop = phase->type(oop);
1239 #ifdef ASSERT
1240 if (oop->is_Con() && toop->is_zero_type() && _type->isa_oopptr()->is_known_instance()) {
1241 // We are not allocated (anymore) and should therefore not have an instance id
1242 dump(1);
1243 assert(false, "Unbuffered inline type should not have known instance id");
1244 }
1245 #endif
1246 const Type* t = toop->filter_speculative(_type);
1247 if (t->singleton()) {
1248 // Don't replace InlineType by a constant
1249 t = _type;
1250 }
1251 const Type* tinit = phase->type(in(IsInit));
1252 if (tinit == Type::TOP) {
1253 return Type::TOP;
1254 }
1255 if (tinit->isa_int() && tinit->is_int()->is_con(1)) {
1256 t = t->join_speculative(TypePtr::NOTNULL);
1257 }
1258 return t;
1259 }