1 /*
2 * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciInlineKlass.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "opto/addnode.hpp"
30 #include "opto/castnode.hpp"
31 #include "opto/graphKit.hpp"
32 #include "opto/inlinetypenode.hpp"
33 #include "opto/rootnode.hpp"
34 #include "opto/phaseX.hpp"
35
36 // Clones the inline type to handle control flow merges involving multiple inline types.
37 // The inputs are replaced by PhiNodes to represent the merged values for the given region.
38 InlineTypeNode* InlineTypeNode::clone_with_phis(PhaseGVN* gvn, Node* region, SafePointNode* map, bool is_init) {
39 InlineTypeNode* vt = clone_if_required(gvn, map);
40 const Type* t = Type::get_const_type(inline_klass());
41 gvn->set_type(vt, t);
42 vt->as_InlineType()->set_type(t);
43
44 // Create a PhiNode for merging the oop values
45 PhiNode* oop = PhiNode::make(region, vt->get_oop(), t);
46 gvn->set_type(oop, t);
47 gvn->record_for_igvn(oop);
48 vt->set_oop(*gvn, oop);
49
50 // Create a PhiNode for merging the is_buffered values
51 t = Type::get_const_basic_type(T_BOOLEAN);
52 Node* is_buffered_node = PhiNode::make(region, vt->get_is_buffered(), t);
53 gvn->set_type(is_buffered_node, t);
54 gvn->record_for_igvn(is_buffered_node);
55 vt->set_req(IsBuffered, is_buffered_node);
56
57 // Create a PhiNode for merging the is_init values
58 Node* is_init_node;
59 if (is_init) {
60 is_init_node = gvn->intcon(1);
61 } else {
62 t = Type::get_const_basic_type(T_BOOLEAN);
63 is_init_node = PhiNode::make(region, vt->get_is_init(), t);
64 gvn->set_type(is_init_node, t);
65 gvn->record_for_igvn(is_init_node);
66 }
67 vt->set_req(IsInit, is_init_node);
68
69 // Create a PhiNode each for merging the field values
70 for (uint i = 0; i < vt->field_count(); ++i) {
71 ciType* type = vt->field_type(i);
72 Node* value = vt->field_value(i);
73 // We limit scalarization for inline types with circular fields and can therefore observe nodes
74 // of the same type but with different scalarization depth during GVN. To avoid inconsistencies
75 // during merging, make sure that we only create Phis for fields that are guaranteed to be scalarized.
76 bool no_circularity = !gvn->C->has_circular_inline_type() || field_is_flat(i);
77 if (value->is_InlineType() && no_circularity) {
78 // Handle inline type fields recursively
79 value = value->as_InlineType()->clone_with_phis(gvn, region, map);
80 } else {
81 t = Type::get_const_type(type);
82 value = PhiNode::make(region, value, t);
83 gvn->set_type(value, t);
84 gvn->record_for_igvn(value);
85 }
86 vt->set_field_value(i, value);
87 }
88 gvn->record_for_igvn(vt);
89 return vt;
90 }
91
92 // Checks if the inputs of the InlineTypeNode were replaced by PhiNodes
93 // for the given region (see InlineTypeNode::clone_with_phis).
94 bool InlineTypeNode::has_phi_inputs(Node* region) {
95 // Check oop input
96 bool result = get_oop()->is_Phi() && get_oop()->as_Phi()->region() == region;
97 #ifdef ASSERT
98 if (result) {
99 // Check all field value inputs for consistency
100 for (uint i = Values; i < field_count(); ++i) {
101 Node* n = in(i);
102 if (n->is_InlineType()) {
103 assert(n->as_InlineType()->has_phi_inputs(region), "inconsistent phi inputs");
104 } else {
105 assert(n->is_Phi() && n->as_Phi()->region() == region, "inconsistent phi inputs");
106 }
107 }
108 }
109 #endif
110 return result;
111 }
112
113 // Merges 'this' with 'other' by updating the input PhiNodes added by 'clone_with_phis'
114 InlineTypeNode* InlineTypeNode::merge_with(PhaseGVN* gvn, const InlineTypeNode* other, int pnum, bool transform) {
115 // Merge oop inputs
116 PhiNode* phi = get_oop()->as_Phi();
117 phi->set_req(pnum, other->get_oop());
118 if (transform) {
119 set_oop(*gvn, gvn->transform(phi));
120 }
121
122 // Merge is_buffered inputs
123 phi = get_is_buffered()->as_Phi();
124 phi->set_req(pnum, other->get_is_buffered());
125 if (transform) {
126 set_req(IsBuffered, gvn->transform(phi));
127 }
128
129 // Merge is_init inputs
130 Node* is_init = get_is_init();
131 if (is_init->is_Phi()) {
132 phi = is_init->as_Phi();
133 phi->set_req(pnum, other->get_is_init());
134 if (transform) {
135 set_req(IsInit, gvn->transform(phi));
136 }
137 } else {
138 assert(is_init->find_int_con(0) == 1, "only with a non null inline type");
139 }
140
141 // Merge field values
142 for (uint i = 0; i < field_count(); ++i) {
143 Node* val1 = field_value(i);
144 Node* val2 = other->field_value(i);
145 if (val1->is_InlineType()) {
146 if (val2->is_Phi()) {
147 val2 = gvn->transform(val2);
148 }
149 val1->as_InlineType()->merge_with(gvn, val2->as_InlineType(), pnum, transform);
150 } else {
151 assert(val1->is_Phi(), "must be a phi node");
152 val1->set_req(pnum, val2);
153 }
154 if (transform) {
155 set_field_value(i, gvn->transform(val1));
156 }
157 }
158 return this;
159 }
160
161 // Adds a new merge path to an inline type node with phi inputs
162 void InlineTypeNode::add_new_path(Node* region) {
163 assert(has_phi_inputs(region), "must have phi inputs");
164
165 PhiNode* phi = get_oop()->as_Phi();
166 phi->add_req(nullptr);
167 assert(phi->req() == region->req(), "must be same size as region");
168
169 phi = get_is_buffered()->as_Phi();
170 phi->add_req(nullptr);
171 assert(phi->req() == region->req(), "must be same size as region");
172
173 phi = get_is_init()->as_Phi();
174 phi->add_req(nullptr);
175 assert(phi->req() == region->req(), "must be same size as region");
176
177 for (uint i = 0; i < field_count(); ++i) {
178 Node* val = field_value(i);
179 if (val->is_InlineType()) {
180 val->as_InlineType()->add_new_path(region);
181 } else {
182 val->as_Phi()->add_req(nullptr);
183 assert(val->req() == region->req(), "must be same size as region");
184 }
185 }
186 }
187
188 Node* InlineTypeNode::field_value(uint index) const {
189 assert(index < field_count(), "index out of bounds");
190 return in(Values + index);
191 }
192
193 // Get the value of the field at the given offset.
194 // If 'recursive' is true, flat inline type fields will be resolved recursively.
195 Node* InlineTypeNode::field_value_by_offset(int offset, bool recursive) const {
196 // If the field at 'offset' belongs to a flat inline type field, 'index' refers to the
197 // corresponding InlineTypeNode input and 'sub_offset' is the offset in flattened inline type.
198 int index = inline_klass()->field_index_by_offset(offset);
199 int sub_offset = offset - field_offset(index);
200 Node* value = field_value(index);
201 assert(value != nullptr, "field value not found");
202 if (recursive && value->is_InlineType()) {
203 if (field_is_flat(index)) {
204 // Flat inline type field
205 InlineTypeNode* vt = value->as_InlineType();
206 sub_offset += vt->inline_klass()->first_field_offset(); // Add header size
207 return vt->field_value_by_offset(sub_offset, recursive);
208 } else {
209 assert(sub_offset == 0, "should not have a sub offset");
210 return value;
211 }
212 }
213 assert(!(recursive && value->is_InlineType()), "should not be an inline type");
214 assert(sub_offset == 0, "offset mismatch");
215 return value;
216 }
217
218 void InlineTypeNode::set_field_value(uint index, Node* value) {
219 assert(index < field_count(), "index out of bounds");
220 set_req(Values + index, value);
221 }
222
223 void InlineTypeNode::set_field_value_by_offset(int offset, Node* value) {
224 set_field_value(field_index(offset), value);
225 }
226
227 int InlineTypeNode::field_offset(uint index) const {
228 assert(index < field_count(), "index out of bounds");
229 return inline_klass()->declared_nonstatic_field_at(index)->offset_in_bytes();
230 }
231
232 uint InlineTypeNode::field_index(int offset) const {
233 uint i = 0;
234 for (; i < field_count() && field_offset(i) != offset; i++) { }
235 assert(i < field_count(), "field not found");
236 return i;
237 }
238
239 ciType* InlineTypeNode::field_type(uint index) const {
240 assert(index < field_count(), "index out of bounds");
241 return inline_klass()->declared_nonstatic_field_at(index)->type();
242 }
243
244 bool InlineTypeNode::field_is_flat(uint index) const {
245 assert(index < field_count(), "index out of bounds");
246 ciField* field = inline_klass()->declared_nonstatic_field_at(index);
247 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
248 return field->is_flat();
249 }
250
251 bool InlineTypeNode::field_is_null_free(uint index) const {
252 assert(index < field_count(), "index out of bounds");
253 ciField* field = inline_klass()->declared_nonstatic_field_at(index);
254 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
255 return field->is_null_free();
256 }
257
258 void InlineTypeNode::make_scalar_in_safepoint(PhaseIterGVN* igvn, Unique_Node_List& worklist, SafePointNode* sfpt) {
259 // Don't scalarize larvals in their own constructor call because the constructor will update them
260 if (is_larval() && sfpt->is_CallJava() && sfpt->as_CallJava()->method() != nullptr && sfpt->as_CallJava()->method()->is_object_constructor() &&
261 sfpt->as_CallJava()->method()->holder()->is_inlinetype() && sfpt->in(TypeFunc::Parms) == this) {
262 assert(is_allocated(igvn), "receiver must be allocated");
263 return;
264 }
265
266 ciInlineKlass* vk = inline_klass();
267 uint nfields = vk->nof_nonstatic_fields();
268 JVMState* jvms = sfpt->jvms();
269 // Replace safepoint edge by SafePointScalarObjectNode and add field values
270 assert(jvms != nullptr, "missing JVMS");
271 uint first_ind = (sfpt->req() - jvms->scloff());
272 SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(type()->isa_instptr(),
273 nullptr,
274 first_ind, nfields);
275 sobj->init_req(0, igvn->C->root());
276 // Nullable inline types have an IsInit field that needs
277 // to be checked before using the field values.
278 if (!igvn->type(get_is_init())->is_int()->is_con(1)) {
279 sfpt->add_req(get_is_init());
280 } else {
281 sfpt->add_req(igvn->C->top());
282 }
283 // Iterate over the inline type fields in order of increasing
284 // offset and add the field values to the safepoint.
285 for (uint j = 0; j < nfields; ++j) {
286 int offset = vk->nonstatic_field_at(j)->offset_in_bytes();
287 Node* value = field_value_by_offset(offset, true /* include flat inline type fields */);
288 if (value->is_InlineType()) {
289 // Add inline type field to the worklist to process later
290 worklist.push(value);
291 }
292 sfpt->add_req(value);
293 }
294 jvms->set_endoff(sfpt->req());
295 sobj = igvn->transform(sobj)->as_SafePointScalarObject();
296 igvn->rehash_node_delayed(sfpt);
297 for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) {
298 Node* debug = sfpt->in(i);
299 if (debug != nullptr && debug->uncast() == this) {
300 sfpt->set_req(i, sobj);
301 }
302 }
303 }
304
305 void InlineTypeNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop) {
306 // If the inline type has a constant or loaded oop, use the oop instead of scalarization
307 // in the safepoint to avoid keeping field loads live just for the debug info.
308 Node* oop = get_oop();
309 // TODO 8325106
310 // TestBasicFunctionality::test3 fails without this. Add more tests?
311 // Add proj nodes here? Recursive handling of phis required? We need a test that fails without.
312 bool use_oop = false;
313 if (allow_oop && is_allocated(igvn) && oop->is_Phi()) {
314 Unique_Node_List worklist;
315 worklist.push(oop);
316 use_oop = true;
317 while (worklist.size() > 0 && use_oop) {
318 Node* n = worklist.pop();
319 for (uint i = 1; i < n->req(); i++) {
320 Node* in = n->in(i);
321 if (in->is_Phi()) {
322 worklist.push(in);
323 // TestNullableArrays.test123 fails when enabling this, probably we should make sure that we don't load from a just allocated object
324 //} else if (!(in->is_Con() || in->is_Parm() || in->is_Load() || (in->isa_DecodeN() && in->in(1)->is_Load()))) {
325 } else if (!(in->is_Con() || in->is_Parm())) {
326 use_oop = false;
327 break;
328 }
329 }
330 }
331 } else {
332 use_oop = allow_oop && is_allocated(igvn) &&
333 (oop->is_Con() || oop->is_Parm() || oop->is_Load() || (oop->isa_DecodeN() && oop->in(1)->is_Load()));
334 }
335
336 ResourceMark rm;
337 Unique_Node_List safepoints;
338 Unique_Node_List vt_worklist;
339 Unique_Node_List worklist;
340 worklist.push(this);
341 while (worklist.size() > 0) {
342 Node* n = worklist.pop();
343 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
344 Node* use = n->fast_out(i);
345 if (use->is_SafePoint() && !use->is_CallLeaf() && (!use->is_Call() || use->as_Call()->has_debug_use(n))) {
346 safepoints.push(use);
347 } else if (use->is_ConstraintCast()) {
348 worklist.push(use);
349 }
350 }
351 }
352
353 // Process all safepoint uses and scalarize inline type
354 while (safepoints.size() > 0) {
355 SafePointNode* sfpt = safepoints.pop()->as_SafePoint();
356 if (use_oop) {
357 for (uint i = sfpt->jvms()->debug_start(); i < sfpt->jvms()->debug_end(); i++) {
358 Node* debug = sfpt->in(i);
359 if (debug != nullptr && debug->uncast() == this) {
360 sfpt->set_req(i, get_oop());
361 }
362 }
363 igvn->rehash_node_delayed(sfpt);
364 } else {
365 make_scalar_in_safepoint(igvn, vt_worklist, sfpt);
366 }
367 }
368 // Now scalarize non-flat fields
369 for (uint i = 0; i < vt_worklist.size(); ++i) {
370 InlineTypeNode* vt = vt_worklist.at(i)->isa_InlineType();
371 vt->make_scalar_in_safepoints(igvn);
372 }
373 if (outcnt() == 0) {
374 igvn->_worklist.push(this);
375 }
376 }
377
378 const TypePtr* InlineTypeNode::field_adr_type(Node* base, int offset, ciInstanceKlass* holder, DecoratorSet decorators, PhaseGVN& gvn) const {
379 const TypeAryPtr* ary_type = gvn.type(base)->isa_aryptr();
380 const TypePtr* adr_type = nullptr;
381 bool is_array = ary_type != nullptr;
382 if ((decorators & C2_MISMATCHED) != 0) {
383 adr_type = TypeRawPtr::BOTTOM;
384 } else if (is_array) {
385 // In the case of a flat inline type array, each field has its own slice
386 adr_type = ary_type->with_field_offset(offset)->add_offset(Type::OffsetBot);
387 } else {
388 ciField* field = holder->get_field_by_offset(offset, false);
389 assert(field != nullptr, "field not found");
390 adr_type = gvn.C->alias_type(field)->adr_type();
391 }
392 return adr_type;
393 }
394
395 // We limit scalarization for inline types with circular fields and can therefore observe nodes
396 // of the same type but with different scalarization depth during GVN. This method adjusts the
397 // scalarization depth to avoid inconsistencies during merging.
398 InlineTypeNode* InlineTypeNode::adjust_scalarization_depth(GraphKit* kit) {
399 if (!kit->C->has_circular_inline_type()) {
400 return this;
401 }
402 GrowableArray<ciType*> visited;
403 visited.push(inline_klass());
404 return adjust_scalarization_depth_impl(kit, visited);
405 }
406
407 InlineTypeNode* InlineTypeNode::adjust_scalarization_depth_impl(GraphKit* kit, GrowableArray<ciType*>& visited) {
408 InlineTypeNode* val = this;
409 for (uint i = 0; i < field_count(); ++i) {
410 Node* value = field_value(i);
411 Node* new_value = value;
412 ciType* ft = field_type(i);
413 if (value->is_InlineType()) {
414 if (!field_is_flat(i) && visited.contains(ft)) {
415 new_value = value->as_InlineType()->buffer(kit)->get_oop();
416 } else {
417 int old_len = visited.length();
418 visited.push(ft);
419 new_value = value->as_InlineType()->adjust_scalarization_depth_impl(kit, visited);
420 visited.trunc_to(old_len);
421 }
422 } else if (ft->is_inlinetype() && !visited.contains(ft)) {
423 int old_len = visited.length();
424 visited.push(ft);
425 new_value = make_from_oop_impl(kit, value, ft->as_inline_klass(), field_is_null_free(i), visited);
426 visited.trunc_to(old_len);
427 }
428 if (value != new_value) {
429 if (val == this) {
430 val = clone_if_required(&kit->gvn(), kit->map());
431 }
432 val->set_field_value(i, new_value);
433 }
434 }
435 return (val == this) ? this : kit->gvn().transform(val)->as_InlineType();
436 }
437
438 void InlineTypeNode::load(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, GrowableArray<ciType*>& visited, int holder_offset, DecoratorSet decorators) {
439 // Initialize the inline type by loading its field values from
440 // memory and adding the values as input edges to the node.
441 for (uint i = 0; i < field_count(); ++i) {
442 int offset = holder_offset + field_offset(i);
443 Node* value = nullptr;
444 ciType* ft = field_type(i);
445 bool null_free = field_is_null_free(i);
446 if (null_free && ft->as_inline_klass()->is_empty()) {
447 // Loading from a field of an empty inline type. Just return the default instance.
448 value = make_default_impl(kit->gvn(), ft->as_inline_klass(), visited);
449 } else if (field_is_flat(i)) {
450 // Recursively load the flat inline type field
451 value = make_from_flat_impl(kit, ft->as_inline_klass(), base, ptr, holder, offset, decorators, visited);
452 } else {
453 const TypeOopPtr* oop_ptr = kit->gvn().type(base)->isa_oopptr();
454 bool is_array = (oop_ptr->isa_aryptr() != nullptr);
455 bool mismatched = (decorators & C2_MISMATCHED) != 0;
456 if (base->is_Con() && !is_array && !mismatched) {
457 // If the oop to the inline type is constant (static final field), we can
458 // also treat the fields as constants because the inline type is immutable.
459 ciObject* constant_oop = oop_ptr->const_oop();
460 ciField* field = holder->get_field_by_offset(offset, false);
461 assert(field != nullptr, "field not found");
462 ciConstant constant = constant_oop->as_instance()->field_value(field);
463 const Type* con_type = Type::make_from_constant(constant, /*require_const=*/ true);
464 assert(con_type != nullptr, "type not found");
465 value = kit->gvn().transform(kit->makecon(con_type));
466 // Check type of constant which might be more precise than the static field type
467 if (con_type->is_inlinetypeptr() && !con_type->is_zero_type()) {
468 ft = con_type->inline_klass();
469 null_free = true;
470 }
471 } else {
472 // Load field value from memory
473 const TypePtr* adr_type = field_adr_type(base, offset, holder, decorators, kit->gvn());
474 Node* adr = kit->basic_plus_adr(base, ptr, offset);
475 BasicType bt = type2field[ft->basic_type()];
476 assert(is_java_primitive(bt) || adr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
477 const Type* val_type = Type::get_const_type(ft);
478 value = kit->access_load_at(base, adr, adr_type, val_type, bt, is_array ? (decorators | IS_ARRAY) : decorators);
479 }
480 // Loading a non-flattened inline type from memory
481 if (visited.contains(ft)) {
482 kit->C->set_has_circular_inline_type(true);
483 } else if (ft->is_inlinetype()) {
484 int old_len = visited.length();
485 visited.push(ft);
486 value = make_from_oop_impl(kit, value, ft->as_inline_klass(), null_free, visited);
487 visited.trunc_to(old_len);
488 }
489 }
490 set_field_value(i, value);
491 }
492 }
493
494 void InlineTypeNode::store_flat(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators) const {
495 if (kit->gvn().type(base)->isa_aryptr()) {
496 kit->C->set_flat_accesses();
497 }
498 // The inline type is embedded into the object without an oop header. Subtract the
499 // offset of the first field to account for the missing header when storing the values.
500 if (holder == nullptr) {
501 holder = inline_klass();
502 }
503 holder_offset -= inline_klass()->first_field_offset();
504 store(kit, base, ptr, holder, holder_offset, decorators);
505 }
506
507 void InlineTypeNode::store(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators, int offsetOnly) const {
508 // Write field values to memory
509 for (uint i = 0; i < field_count(); ++i) {
510 if (offsetOnly != -1 && offsetOnly != field_offset(i)) continue;
511 int offset = holder_offset + field_offset(i);
512 Node* value = field_value(i);
513 ciType* ft = field_type(i);
514 if (field_is_flat(i)) {
515 // Recursively store the flat inline type field
516 value->as_InlineType()->store_flat(kit, base, ptr, holder, offset, decorators);
517 } else {
518 // Store field value to memory
519 const TypePtr* adr_type = field_adr_type(base, offset, holder, decorators, kit->gvn());
520 Node* adr = kit->basic_plus_adr(base, ptr, offset);
521 BasicType bt = type2field[ft->basic_type()];
522 assert(is_java_primitive(bt) || adr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
523 const Type* val_type = Type::get_const_type(ft);
524 bool is_array = (kit->gvn().type(base)->isa_aryptr() != nullptr);
525 kit->access_store_at(base, adr, adr_type, value, val_type, bt, is_array ? (decorators | IS_ARRAY) : decorators);
526 }
527 }
528 }
529
530 InlineTypeNode* InlineTypeNode::buffer(GraphKit* kit, bool safe_for_replace) {
531 if (kit->gvn().find_int_con(get_is_buffered(), 0) == 1) {
532 // Already buffered
533 return this;
534 }
535
536 // Check if inline type is already buffered
537 Node* not_buffered_ctl = kit->top();
538 Node* not_null_oop = kit->null_check_oop(get_oop(), ¬_buffered_ctl, /* never_see_null = */ false, safe_for_replace);
539 if (not_buffered_ctl->is_top()) {
540 // Already buffered
541 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map(), safe_for_replace);
542 vt->set_is_buffered(kit->gvn());
543 vt = kit->gvn().transform(vt)->as_InlineType();
544 if (safe_for_replace) {
545 kit->replace_in_map(this, vt);
546 }
547 return vt;
548 }
549 Node* buffered_ctl = kit->control();
550 kit->set_control(not_buffered_ctl);
551
552 // Inline type is not buffered, check if it is null.
553 Node* null_ctl = kit->top();
554 kit->null_check_common(get_is_init(), T_INT, false, &null_ctl);
555 bool null_free = null_ctl->is_top();
556
557 RegionNode* region = new RegionNode(4);
558 PhiNode* oop = PhiNode::make(region, not_null_oop, type()->join_speculative(null_free ? TypePtr::NOTNULL : TypePtr::BOTTOM));
559
560 // InlineType is already buffered
561 region->init_req(1, buffered_ctl);
562 oop->init_req(1, not_null_oop);
563
564 // InlineType is null
565 region->init_req(2, null_ctl);
566 oop->init_req(2, kit->gvn().zerocon(T_OBJECT));
567
568 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO);
569 PhiNode* mem = PhiNode::make(region, kit->merged_memory(), Type::MEMORY, TypePtr::BOTTOM);
570
571 int bci = kit->bci();
572 bool reexecute = kit->jvms()->should_reexecute();
573 if (!kit->stopped()) {
574 assert(!is_allocated(&kit->gvn()), "already buffered");
575
576 // Allocate and initialize buffer
577 PreserveJVMState pjvms(kit);
578 // Propagate re-execution state and bci
579 kit->set_bci(bci);
580 kit->jvms()->set_bci(bci);
581 kit->jvms()->set_should_reexecute(reexecute);
582
583 kit->kill_dead_locals();
584 ciInlineKlass* vk = inline_klass();
585 Node* klass_node = kit->makecon(TypeKlassPtr::make(vk));
586 Node* alloc_oop = kit->new_instance(klass_node, nullptr, nullptr, /* deoptimize_on_exception */ true, this);
587 // No need to initialize a larval buffer, we make sure that the oop can not escape
588 if (!is_larval()) {
589 // Larval will be initialized later
590 // TODO 8325106 should this use C2_TIGHTLY_COUPLED_ALLOC?
591 store(kit, alloc_oop, alloc_oop, vk);
592
593 // Do not let stores that initialize this buffer be reordered with a subsequent
594 // store that would make this buffer accessible by other threads.
595 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_oop);
596 assert(alloc != nullptr, "must have an allocation node");
597 kit->insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress));
598 }
599
600 region->init_req(3, kit->control());
601 oop ->init_req(3, alloc_oop);
602 io ->init_req(3, kit->i_o());
603 mem ->init_req(3, kit->merged_memory());
604 }
605
606 // Update GraphKit
607 kit->set_control(kit->gvn().transform(region));
608 kit->set_i_o(kit->gvn().transform(io));
609 kit->set_all_memory(kit->gvn().transform(mem));
610 kit->record_for_igvn(region);
611 kit->record_for_igvn(oop);
612 kit->record_for_igvn(io);
613 kit->record_for_igvn(mem);
614
615 // Use cloned InlineTypeNode to propagate oop from now on
616 Node* res_oop = kit->gvn().transform(oop);
617 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map(), safe_for_replace);
618 vt->set_oop(kit->gvn(), res_oop);
619 vt->set_is_buffered(kit->gvn());
620 vt = kit->gvn().transform(vt)->as_InlineType();
621 if (safe_for_replace) {
622 kit->replace_in_map(this, vt);
623 }
624 // InlineTypeNode::remove_redundant_allocations piggybacks on split if.
625 // Make sure it gets a chance to remove this allocation.
626 kit->C->set_has_split_ifs(true);
627 return vt;
628 }
629
630 bool InlineTypeNode::is_allocated(PhaseGVN* phase) const {
631 if (phase->find_int_con(get_is_buffered(), 0) == 1) {
632 return true;
633 }
634 Node* oop = get_oop();
635 const Type* oop_type = (phase != nullptr) ? phase->type(oop) : oop->bottom_type();
636 return !oop_type->maybe_null();
637 }
638
639 // When a call returns multiple values, it has several result
640 // projections, one per field. Replacing the result of the call by an
641 // inline type node (after late inlining) requires that for each result
642 // projection, we find the corresponding inline type field.
643 void InlineTypeNode::replace_call_results(GraphKit* kit, CallNode* call, Compile* C) {
644 ciInlineKlass* vk = inline_klass();
645 for (DUIterator_Fast imax, i = call->fast_outs(imax); i < imax; i++) {
646 ProjNode* pn = call->fast_out(i)->as_Proj();
647 uint con = pn->_con;
648 Node* field = nullptr;
649 if (con == TypeFunc::Parms) {
650 field = get_oop();
651 } else if (con == (call->tf()->range_cc()->cnt() - 1)) {
652 field = get_is_init();
653 } else if (con > TypeFunc::Parms) {
654 uint field_nb = con - (TypeFunc::Parms+1);
655 int extra = 0;
656 for (uint j = 0; j < field_nb - extra; j++) {
657 ciField* f = vk->nonstatic_field_at(j);
658 BasicType bt = f->type()->basic_type();
659 if (bt == T_LONG || bt == T_DOUBLE) {
660 extra++;
661 }
662 }
663 ciField* f = vk->nonstatic_field_at(field_nb - extra);
664 field = field_value_by_offset(f->offset_in_bytes(), true);
665 }
666 if (field != nullptr) {
667 C->gvn_replace_by(pn, field);
668 C->initial_gvn()->hash_delete(pn);
669 pn->set_req(0, C->top());
670 --i; --imax;
671 }
672 }
673 }
674
675 Node* InlineTypeNode::allocate_fields(GraphKit* kit) {
676 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map());
677 for (uint i = 0; i < field_count(); i++) {
678 Node* value = field_value(i);
679 if (field_is_flat(i)) {
680 // Flat inline type field
681 vt->set_field_value(i, value->as_InlineType()->allocate_fields(kit));
682 } else if (value->is_InlineType()) {
683 // Non-flat inline type field
684 vt->set_field_value(i, value->as_InlineType()->buffer(kit));
685 }
686 }
687 vt = kit->gvn().transform(vt)->as_InlineType();
688 kit->replace_in_map(this, vt);
689 return vt;
690 }
691
692 // Replace a buffer allocation by a dominating allocation
693 static void replace_allocation(PhaseIterGVN* igvn, Node* res, Node* dom) {
694 // Remove initializing stores and GC barriers
695 for (DUIterator_Fast imax, i = res->fast_outs(imax); i < imax; i++) {
696 Node* use = res->fast_out(i);
697 if (use->is_AddP()) {
698 for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) {
699 Node* store = use->fast_out(j)->isa_Store();
700 if (store != nullptr) {
701 igvn->rehash_node_delayed(store);
702 igvn->replace_in_uses(store, store->in(MemNode::Memory));
703 }
704 }
705 } else if (use->Opcode() == Op_CastP2X) {
706 if (UseG1GC && use->find_out_with(Op_XorX)->in(1) != use) {
707 // The G1 pre-barrier uses a CastP2X both for the pointer of the object
708 // we store into, as well as the value we are storing. Skip if this is a
709 // barrier for storing 'res' into another object.
710 continue;
711 }
712 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
713 bs->eliminate_gc_barrier(igvn, use);
714 --i; --imax;
715 }
716 }
717 igvn->replace_node(res, dom);
718 }
719
720 Node* InlineTypeNode::Ideal(PhaseGVN* phase, bool can_reshape) {
721 Node* oop = get_oop();
722 const Type* tinit = phase->type(get_is_init());
723 if (!is_larval(phase) && !is_larval() &&
724 (tinit->isa_int() && tinit->is_int()->is_con(1)) &&
725 (is_default(phase) || inline_klass()->is_empty()) &&
726 inline_klass()->is_initialized() &&
727 (!oop->is_Con() || phase->type(oop)->is_zero_type())) {
728 // Use the pre-allocated oop for null-free default or empty inline types
729 set_oop(*phase, default_oop(*phase, inline_klass()));
730 assert(is_allocated(phase), "should now be allocated");
731 return this;
732 }
733 if (oop->isa_InlineType() && !phase->type(oop)->maybe_null()) {
734 InlineTypeNode* vtptr = oop->as_InlineType();
735 set_oop(*phase, vtptr->get_oop());
736 set_is_buffered(*phase);
737 set_is_init(*phase);
738 for (uint i = Values; i < vtptr->req(); ++i) {
739 set_req(i, vtptr->in(i));
740 }
741 return this;
742 }
743 // TODO 8325106 Re-evaluate this: We prefer a "loaded" oop because it's free. The existing oop might come from a buffering.
744 if (!is_larval(phase) && !is_larval()) {
745 // Save base oop if fields are loaded from memory and the inline
746 // type is not buffered (in this case we should not use the oop).
747 Node* base = is_loaded(phase);
748 if (base != nullptr && get_oop() != base && !phase->type(base)->maybe_null()) {
749 set_oop(*phase, base);
750 assert(is_allocated(phase), "should now be allocated");
751 return this;
752 }
753 }
754
755 if (can_reshape) {
756 PhaseIterGVN* igvn = phase->is_IterGVN();
757 if (is_allocated(phase)) {
758 // Search for and remove re-allocations of this inline type. Ignore scalar replaceable ones,
759 // they will be removed anyway and changing the memory chain will confuse other optimizations.
760 // This can happen with late inlining when we first allocate an inline type argument
761 // but later decide to inline the call after the callee code also triggered allocation.
762 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
763 AllocateNode* alloc = fast_out(i)->isa_Allocate();
764 if (alloc != nullptr && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable) {
765 // Found a re-allocation
766 Node* res = alloc->result_cast();
767 if (res != nullptr && res->is_CheckCastPP()) {
768 // Replace allocation by oop and unlink AllocateNode
769 replace_allocation(igvn, res, oop);
770 igvn->replace_input_of(alloc, AllocateNode::InlineType, igvn->C->top());
771 --i; --imax;
772 }
773 }
774 }
775 }
776 }
777
778 return nullptr;
779 }
780
781 InlineTypeNode* InlineTypeNode::make_uninitialized(PhaseGVN& gvn, ciInlineKlass* vk, bool null_free) {
782 // Create a new InlineTypeNode with uninitialized values and nullptr oop
783 bool use_default_oop = vk->is_empty() && vk->is_initialized() && null_free;
784 Node* oop = use_default_oop ? default_oop(gvn, vk) : gvn.zerocon(T_OBJECT);
785 InlineTypeNode* vt = new InlineTypeNode(vk, oop, null_free);
786 vt->set_is_buffered(gvn, use_default_oop);
787 vt->set_is_init(gvn);
788 return vt;
789 }
790
791 Node* InlineTypeNode::default_oop(PhaseGVN& gvn, ciInlineKlass* vk) {
792 // Returns the constant oop of the default inline type allocation
793 return gvn.makecon(TypeInstPtr::make(vk->default_instance()));
794 }
795
796 InlineTypeNode* InlineTypeNode::make_default(PhaseGVN& gvn, ciInlineKlass* vk, bool is_larval) {
797 GrowableArray<ciType*> visited;
798 visited.push(vk);
799 return make_default_impl(gvn, vk, visited, is_larval);
800 }
801
802 InlineTypeNode* InlineTypeNode::make_default_impl(PhaseGVN& gvn, ciInlineKlass* vk, GrowableArray<ciType*>& visited, bool is_larval) {
803 // Create a new InlineTypeNode with default values
804 Node* oop = vk->is_initialized() && !is_larval ? default_oop(gvn, vk) : gvn.zerocon(T_OBJECT);
805 InlineTypeNode* vt = new InlineTypeNode(vk, oop, /* null_free= */ true);
806 // TODO 8325106 we should be able to set buffered here for non-larvals, right?
807 //vt->set_is_buffered(gvn, vk->is_initialized());
808 vt->set_is_buffered(gvn, false);
809 vt->set_is_init(gvn);
810 vt->set_is_larval(is_larval);
811 for (uint i = 0; i < vt->field_count(); ++i) {
812 ciType* ft = vt->field_type(i);
813 Node* value = gvn.zerocon(ft->basic_type());
814 if (!vt->field_is_flat(i) && visited.contains(ft)) {
815 gvn.C->set_has_circular_inline_type(true);
816 } else if (ft->is_inlinetype()) {
817 int old_len = visited.length();
818 visited.push(ft);
819 ciInlineKlass* vk = ft->as_inline_klass();
820 if (vt->field_is_null_free(i)) {
821 value = make_default_impl(gvn, vk, visited);
822 } else {
823 value = make_null_impl(gvn, vk, visited);
824 }
825 visited.trunc_to(old_len);
826 }
827 vt->set_field_value(i, value);
828 }
829 vt = gvn.transform(vt)->as_InlineType();
830 assert(vt->is_default(&gvn), "must be the default inline type");
831 return vt;
832 }
833
834 bool InlineTypeNode::is_default(PhaseGVN* gvn) const {
835 const Type* tinit = gvn->type(get_is_init());
836 if (!tinit->isa_int() || !tinit->is_int()->is_con(1)) {
837 return false; // May be null
838 }
839 for (uint i = 0; i < field_count(); ++i) {
840 ciType* ft = field_type(i);
841 Node* value = field_value(i);
842 if (field_is_null_free(i)) {
843 if (!value->is_InlineType() || !value->as_InlineType()->is_default(gvn)) {
844 return false;
845 }
846 continue;
847 } else if (value->is_InlineType()) {
848 if (value->as_InlineType()->is_default(gvn)) {
849 continue;
850 } else {
851 const Type* tinit = gvn->type(value->as_InlineType()->get_is_init());
852 if (tinit->isa_int() && tinit->is_int()->is_con(0)) {
853 continue;
854 }
855 return false;
856 }
857 }
858 if (!gvn->type(value)->is_zero_type()) {
859 return false;
860 }
861 }
862 return true;
863 }
864
865 InlineTypeNode* InlineTypeNode::make_from_oop(GraphKit* kit, Node* oop, ciInlineKlass* vk, bool null_free, bool is_larval) {
866 GrowableArray<ciType*> visited;
867 visited.push(vk);
868 return make_from_oop_impl(kit, oop, vk, null_free, visited, is_larval);
869 }
870
871 InlineTypeNode* InlineTypeNode::make_from_oop_impl(GraphKit* kit, Node* oop, ciInlineKlass* vk, bool null_free, GrowableArray<ciType*>& visited, bool is_larval) {
872 PhaseGVN& gvn = kit->gvn();
873
874 if (!is_larval && vk->is_empty() && null_free) {
875 InlineTypeNode* def = make_default_impl(gvn, vk, visited);
876 kit->record_for_igvn(def);
877 return def;
878 }
879 // Create and initialize an InlineTypeNode by loading all field
880 // values from a heap-allocated version and also save the oop.
881 InlineTypeNode* vt = nullptr;
882
883 if (oop->isa_InlineType()) {
884 assert(!is_larval || oop->as_InlineType()->is_larval(), "must be larval");
885 return oop->as_InlineType();
886 } else if (gvn.type(oop)->maybe_null()) {
887 // Add a null check because the oop may be null
888 Node* null_ctl = kit->top();
889 Node* not_null_oop = kit->null_check_oop(oop, &null_ctl);
890 if (kit->stopped()) {
891 // Constant null
892 kit->set_control(null_ctl);
893 if (null_free) {
894 vt = make_default_impl(gvn, vk, visited);
895 } else {
896 vt = make_null_impl(gvn, vk, visited);
897 }
898 kit->record_for_igvn(vt);
899 return vt;
900 }
901 vt = new InlineTypeNode(vk, not_null_oop, null_free);
902 vt->set_is_buffered(gvn);
903 vt->set_is_init(gvn);
904 vt->set_is_larval(is_larval);
905 vt->load(kit, not_null_oop, not_null_oop, vk, visited);
906
907 if (null_ctl != kit->top()) {
908 InlineTypeNode* null_vt = nullptr;
909 if (null_free) {
910 null_vt = make_default_impl(gvn, vk, visited);
911 } else {
912 null_vt = make_null_impl(gvn, vk, visited);
913 }
914 Node* region = new RegionNode(3);
915 region->init_req(1, kit->control());
916 region->init_req(2, null_ctl);
917 vt = vt->clone_with_phis(&gvn, region, kit->map());
918 vt->merge_with(&gvn, null_vt, 2, true);
919 if (!null_free) {
920 vt->set_oop(gvn, oop);
921 }
922 kit->set_control(gvn.transform(region));
923 }
924 } else {
925 // Oop can never be null
926 vt = new InlineTypeNode(vk, oop, /* null_free= */ true);
927 Node* init_ctl = kit->control();
928 vt->set_is_buffered(gvn);
929 vt->set_is_init(gvn);
930 vt->set_is_larval(is_larval);
931 vt->load(kit, oop, oop, vk, visited);
932 // TODO 8284443
933 // assert(!null_free || vt->as_InlineType()->is_default(&gvn) || init_ctl != kit->control() || !gvn.type(oop)->is_inlinetypeptr() || oop->is_Con() || oop->Opcode() == Op_InlineType ||
934 // AllocateNode::Ideal_allocation(oop, &gvn) != nullptr || vt->as_InlineType()->is_loaded(&gvn) == oop, "inline type should be loaded");
935 }
936 assert(vt->is_allocated(&gvn) || (null_free && !vk->is_initialized()), "inline type should be allocated");
937 kit->record_for_igvn(vt);
938 return gvn.transform(vt)->as_InlineType();
939 }
940
941 InlineTypeNode* InlineTypeNode::make_from_flat(GraphKit* kit, ciInlineKlass* vk, Node* obj, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators) {
942 GrowableArray<ciType*> visited;
943 visited.push(vk);
944 return make_from_flat_impl(kit, vk, obj, ptr, holder, holder_offset, decorators, visited);
945 }
946
947 // GraphKit wrapper for the 'make_from_flat' method
948 InlineTypeNode* InlineTypeNode::make_from_flat_impl(GraphKit* kit, ciInlineKlass* vk, Node* obj, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators, GrowableArray<ciType*>& visited) {
949 if (kit->gvn().type(obj)->isa_aryptr()) {
950 kit->C->set_flat_accesses();
951 }
952 // Create and initialize an InlineTypeNode by loading all field values from
953 // a flat inline type field at 'holder_offset' or from an inline type array.
954 InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk);
955 // The inline type is flattened into the object without an oop header. Subtract the
956 // offset of the first field to account for the missing header when loading the values.
957 holder_offset -= vk->first_field_offset();
958 vt->load(kit, obj, ptr, holder, visited, holder_offset, decorators);
959 assert(vt->is_loaded(&kit->gvn()) != obj, "holder oop should not be used as flattened inline type oop");
960 return kit->gvn().transform(vt)->as_InlineType();
961 }
962
963 InlineTypeNode* InlineTypeNode::make_from_multi(GraphKit* kit, MultiNode* multi, ciInlineKlass* vk, uint& base_input, bool in, bool null_free) {
964 InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk, null_free);
965 if (!in) {
966 // Keep track of the oop. The returned inline type might already be buffered.
967 Node* oop = kit->gvn().transform(new ProjNode(multi, base_input++));
968 vt->set_oop(kit->gvn(), oop);
969 }
970 GrowableArray<ciType*> visited;
971 visited.push(vk);
972 vt->initialize_fields(kit, multi, base_input, in, null_free, nullptr, visited);
973 return kit->gvn().transform(vt)->as_InlineType();
974 }
975
976 InlineTypeNode* InlineTypeNode::make_larval(GraphKit* kit, bool allocate) const {
977 ciInlineKlass* vk = inline_klass();
978 InlineTypeNode* res = make_uninitialized(kit->gvn(), vk);
979 for (uint i = 1; i < req(); ++i) {
980 res->set_req(i, in(i));
981 }
982
983 if (allocate) {
984 // Re-execute if buffering triggers deoptimization
985 PreserveReexecuteState preexecs(kit);
986 kit->jvms()->set_should_reexecute(true);
987 Node* klass_node = kit->makecon(TypeKlassPtr::make(vk));
988 Node* alloc_oop = kit->new_instance(klass_node, nullptr, nullptr, true);
989 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_oop);
990 alloc->_larval = true;
991
992 store(kit, alloc_oop, alloc_oop, vk);
993 res->set_oop(kit->gvn(), alloc_oop);
994 }
995 // TODO 8239003
996 //res->set_type(TypeInlineType::make(vk, true));
997 res = kit->gvn().transform(res)->as_InlineType();
998 assert(!allocate || res->is_allocated(&kit->gvn()), "must be allocated");
999 return res;
1000 }
1001
1002 InlineTypeNode* InlineTypeNode::finish_larval(GraphKit* kit) const {
1003 Node* obj = get_oop();
1004 Node* mark_addr = kit->basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
1005 Node* mark = kit->make_load(nullptr, mark_addr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered);
1006 mark = kit->gvn().transform(new AndXNode(mark, kit->MakeConX(~markWord::larval_bit_in_place)));
1007 kit->store_to_memory(kit->control(), mark_addr, mark, TypeX_X->basic_type(), kit->gvn().type(mark_addr)->is_ptr(), MemNode::unordered);
1008
1009 // Do not let stores that initialize this buffer be reordered with a subsequent
1010 // store that would make this buffer accessible by other threads.
1011 AllocateNode* alloc = AllocateNode::Ideal_allocation(obj);
1012 assert(alloc != nullptr, "must have an allocation node");
1013 kit->insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress));
1014
1015 ciInlineKlass* vk = inline_klass();
1016 InlineTypeNode* res = make_uninitialized(kit->gvn(), vk);
1017 for (uint i = 1; i < req(); ++i) {
1018 res->set_req(i, in(i));
1019 }
1020 // TODO 8239003
1021 //res->set_type(TypeInlineType::make(vk, false));
1022 res = kit->gvn().transform(res)->as_InlineType();
1023 return res;
1024 }
1025
1026 bool InlineTypeNode::is_larval(PhaseGVN* gvn) const {
1027 if (!is_allocated(gvn)) {
1028 return false;
1029 }
1030
1031 Node* oop = get_oop();
1032 AllocateNode* alloc = AllocateNode::Ideal_allocation(oop);
1033 return alloc != nullptr && alloc->_larval;
1034 }
1035
1036 Node* InlineTypeNode::is_loaded(PhaseGVN* phase, ciInlineKlass* vk, Node* base, int holder_offset) {
1037 if (vk == nullptr) {
1038 vk = inline_klass();
1039 }
1040 if (field_count() == 0 && vk->is_initialized()) {
1041 const Type* tinit = phase->type(in(IsInit));
1042 // TODO 8325106
1043 if (false && !is_larval() && tinit->isa_int() && tinit->is_int()->is_con(1)) {
1044 assert(is_allocated(phase), "must be allocated");
1045 return get_oop();
1046 } else {
1047 // TODO 8284443
1048 return nullptr;
1049 }
1050 }
1051 for (uint i = 0; i < field_count(); ++i) {
1052 int offset = holder_offset + field_offset(i);
1053 Node* value = field_value(i);
1054 if (value->is_InlineType()) {
1055 InlineTypeNode* vt = value->as_InlineType();
1056 if (vt->type()->inline_klass()->is_empty()) {
1057 continue;
1058 } else if (field_is_flat(i) && vt->is_InlineType()) {
1059 // Check inline type field load recursively
1060 base = vt->as_InlineType()->is_loaded(phase, vk, base, offset - vt->type()->inline_klass()->first_field_offset());
1061 if (base == nullptr) {
1062 return nullptr;
1063 }
1064 continue;
1065 } else {
1066 value = vt->get_oop();
1067 if (value->Opcode() == Op_CastPP) {
1068 // Skip CastPP
1069 value = value->in(1);
1070 }
1071 }
1072 }
1073 if (value->isa_DecodeN()) {
1074 // Skip DecodeN
1075 value = value->in(1);
1076 }
1077 if (value->isa_Load()) {
1078 // Check if base and offset of field load matches inline type layout
1079 intptr_t loffset = 0;
1080 Node* lbase = AddPNode::Ideal_base_and_offset(value->in(MemNode::Address), phase, loffset);
1081 if (lbase == nullptr || (lbase != base && base != nullptr) || loffset != offset) {
1082 return nullptr;
1083 } else if (base == nullptr) {
1084 // Set base and check if pointer type matches
1085 base = lbase;
1086 const TypeInstPtr* vtptr = phase->type(base)->isa_instptr();
1087 if (vtptr == nullptr || !vtptr->instance_klass()->equals(vk)) {
1088 return nullptr;
1089 }
1090 }
1091 } else {
1092 return nullptr;
1093 }
1094 }
1095 return base;
1096 }
1097
1098 Node* InlineTypeNode::tagged_klass(ciInlineKlass* vk, PhaseGVN& gvn) {
1099 const TypeKlassPtr* tk = TypeKlassPtr::make(vk);
1100 intptr_t bits = tk->get_con();
1101 set_nth_bit(bits, 0);
1102 return gvn.longcon((jlong)bits);
1103 }
1104
1105 void InlineTypeNode::pass_fields(GraphKit* kit, Node* n, uint& base_input, bool in, bool null_free) {
1106 if (!null_free && in) {
1107 n->init_req(base_input++, get_is_init());
1108 }
1109 for (uint i = 0; i < field_count(); i++) {
1110 Node* arg = field_value(i);
1111 if (field_is_flat(i)) {
1112 // Flat inline type field
1113 arg->as_InlineType()->pass_fields(kit, n, base_input, in);
1114 } else {
1115 if (arg->is_InlineType()) {
1116 // Non-flat inline type field
1117 InlineTypeNode* vt = arg->as_InlineType();
1118 assert(n->Opcode() != Op_Return || vt->is_allocated(&kit->gvn()), "inline type field should be allocated on return");
1119 arg = vt->buffer(kit);
1120 }
1121 // Initialize call/return arguments
1122 n->init_req(base_input++, arg);
1123 if (field_type(i)->size() == 2) {
1124 n->init_req(base_input++, kit->top());
1125 }
1126 }
1127 }
1128 // The last argument is used to pass IsInit information to compiled code and not required here.
1129 if (!null_free && !in) {
1130 n->init_req(base_input++, kit->top());
1131 }
1132 }
1133
1134 void InlineTypeNode::initialize_fields(GraphKit* kit, MultiNode* multi, uint& base_input, bool in, bool null_free, Node* null_check_region, GrowableArray<ciType*>& visited) {
1135 PhaseGVN& gvn = kit->gvn();
1136 Node* is_init = nullptr;
1137 if (!null_free) {
1138 // Nullable inline type
1139 if (in) {
1140 // Set IsInit field
1141 if (multi->is_Start()) {
1142 is_init = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1143 } else {
1144 is_init = multi->as_Call()->in(base_input);
1145 }
1146 set_req(IsInit, is_init);
1147 base_input++;
1148 }
1149 // Add a null check to make subsequent loads dependent on
1150 assert(null_check_region == nullptr, "already set");
1151 if (is_init == nullptr) {
1152 // Will only be initialized below, use dummy node for now
1153 is_init = new Node(1);
1154 gvn.set_type_bottom(is_init);
1155 }
1156 Node* null_ctrl = kit->top();
1157 kit->null_check_common(is_init, T_INT, false, &null_ctrl);
1158 Node* non_null_ctrl = kit->control();
1159 null_check_region = new RegionNode(3);
1160 null_check_region->init_req(1, non_null_ctrl);
1161 null_check_region->init_req(2, null_ctrl);
1162 null_check_region = gvn.transform(null_check_region);
1163 kit->set_control(null_check_region);
1164 }
1165
1166 for (uint i = 0; i < field_count(); ++i) {
1167 ciType* type = field_type(i);
1168 Node* parm = nullptr;
1169 if (field_is_flat(i)) {
1170 // Flat inline type field
1171 InlineTypeNode* vt = make_uninitialized(gvn, type->as_inline_klass());
1172 vt->initialize_fields(kit, multi, base_input, in, true, null_check_region, visited);
1173 parm = gvn.transform(vt);
1174 } else {
1175 if (multi->is_Start()) {
1176 assert(in, "return from start?");
1177 parm = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1178 } else if (in) {
1179 parm = multi->as_Call()->in(base_input);
1180 } else {
1181 parm = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1182 }
1183 // Non-flat inline type field
1184 if (type->is_inlinetype()) {
1185 if (null_check_region != nullptr) {
1186 // We limit scalarization for inline types with circular fields and can therefore observe nodes
1187 // of the same type but with different scalarization depth during GVN. To avoid inconsistencies
1188 // during merging, make sure that we only create Phis for fields that are guaranteed to be scalarized.
1189 if (parm->is_InlineType() && kit->C->has_circular_inline_type()) {
1190 parm = parm->as_InlineType()->get_oop();
1191 }
1192 // Holder is nullable, set field to nullptr if holder is nullptr to avoid loading from uninitialized memory
1193 parm = PhiNode::make(null_check_region, parm, TypeInstPtr::make(TypePtr::BotPTR, type->as_inline_klass()));
1194 parm->set_req(2, kit->zerocon(T_OBJECT));
1195 parm = gvn.transform(parm);
1196 }
1197 if (visited.contains(type)) {
1198 kit->C->set_has_circular_inline_type(true);
1199 } else if (!parm->is_InlineType()) {
1200 int old_len = visited.length();
1201 visited.push(type);
1202 parm = make_from_oop_impl(kit, parm, type->as_inline_klass(), field_is_null_free(i), visited);
1203 visited.trunc_to(old_len);
1204 }
1205 }
1206 base_input += type->size();
1207 }
1208 assert(parm != nullptr, "should never be null");
1209 assert(field_value(i) == nullptr, "already set");
1210 set_field_value(i, parm);
1211 gvn.record_for_igvn(parm);
1212 }
1213 // The last argument is used to pass IsInit information to compiled code
1214 if (!null_free && !in) {
1215 Node* cmp = is_init->raw_out(0);
1216 is_init = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1217 set_req(IsInit, is_init);
1218 gvn.hash_delete(cmp);
1219 cmp->set_req(1, is_init);
1220 gvn.hash_find_insert(cmp);
1221 base_input++;
1222 }
1223 }
1224
1225 // Search for multiple allocations of this inline type and try to replace them by dominating allocations.
1226 // Equivalent InlineTypeNodes are merged by GVN, so we just need to search for AllocateNode users to find redundant allocations.
1227 void InlineTypeNode::remove_redundant_allocations(PhaseIdealLoop* phase) {
1228 if (is_larval()) {
1229 return;
1230 }
1231 PhaseIterGVN* igvn = &phase->igvn();
1232 // Search for allocations of this inline type. Ignore scalar replaceable ones, they
1233 // will be removed anyway and changing the memory chain will confuse other optimizations.
1234 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1235 AllocateNode* alloc = fast_out(i)->isa_Allocate();
1236 if (alloc != nullptr && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable) {
1237 Node* res = alloc->result_cast();
1238 if (res == nullptr || !res->is_CheckCastPP()) {
1239 break; // No unique CheckCastPP
1240 }
1241 // TODO 8325106
1242 // assert((!is_default(igvn) || !inline_klass()->is_initialized()) && !is_allocated(igvn), "re-allocation should be removed by Ideal transformation");
1243 // Search for a dominating allocation of the same inline type
1244 Node* res_dom = res;
1245 for (DUIterator_Fast jmax, j = fast_outs(jmax); j < jmax; j++) {
1246 AllocateNode* alloc_other = fast_out(j)->isa_Allocate();
1247 if (alloc_other != nullptr && alloc_other->in(AllocateNode::InlineType) == this && !alloc_other->_is_scalar_replaceable) {
1248 Node* res_other = alloc_other->result_cast();
1249 if (res_other != nullptr && res_other->is_CheckCastPP() && res_other != res_dom &&
1250 phase->is_dominator(res_other->in(0), res_dom->in(0))) {
1251 res_dom = res_other;
1252 }
1253 }
1254 }
1255 if (res_dom != res) {
1256 // Replace allocation by dominating one.
1257 replace_allocation(igvn, res, res_dom);
1258 // The result of the dominated allocation is now unused and will be removed
1259 // later in PhaseMacroExpand::eliminate_allocate_node to not confuse loop opts.
1260 igvn->_worklist.push(alloc);
1261 }
1262 }
1263 }
1264 }
1265
1266 InlineTypeNode* InlineTypeNode::make_null(PhaseGVN& gvn, ciInlineKlass* vk, bool transform) {
1267 GrowableArray<ciType*> visited;
1268 visited.push(vk);
1269 return make_null_impl(gvn, vk, visited, transform);
1270 }
1271
1272 InlineTypeNode* InlineTypeNode::make_null_impl(PhaseGVN& gvn, ciInlineKlass* vk, GrowableArray<ciType*>& visited, bool transform) {
1273 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT), /* null_free= */ false);
1274 vt->set_is_buffered(gvn);
1275 vt->set_is_init(gvn, false);
1276 for (uint i = 0; i < vt->field_count(); i++) {
1277 ciType* ft = vt->field_type(i);
1278 Node* value = gvn.zerocon(ft->basic_type());
1279 if (!vt->field_is_flat(i) && visited.contains(ft)) {
1280 gvn.C->set_has_circular_inline_type(true);
1281 } else if (ft->is_inlinetype()) {
1282 int old_len = visited.length();
1283 visited.push(ft);
1284 value = make_null_impl(gvn, ft->as_inline_klass(), visited);
1285 visited.trunc_to(old_len);
1286 }
1287 vt->set_field_value(i, value);
1288 }
1289 return transform ? gvn.transform(vt)->as_InlineType() : vt;
1290 }
1291
1292 InlineTypeNode* InlineTypeNode::clone_if_required(PhaseGVN* gvn, SafePointNode* map, bool safe_for_replace) {
1293 if (!safe_for_replace || (map == nullptr && outcnt() != 0)) {
1294 return clone()->as_InlineType();
1295 }
1296 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1297 if (fast_out(i) != map) {
1298 return clone()->as_InlineType();
1299 }
1300 }
1301 gvn->hash_delete(this);
1302 return this;
1303 }
1304
1305 const Type* InlineTypeNode::Value(PhaseGVN* phase) const {
1306 Node* oop = get_oop();
1307 const Type* toop = phase->type(oop);
1308 #ifdef ASSERT
1309 if (oop->is_Con() && toop->is_zero_type() && _type->isa_oopptr()->is_known_instance()) {
1310 // We are not allocated (anymore) and should therefore not have an instance id
1311 dump(1);
1312 assert(false, "Unbuffered inline type should not have known instance id");
1313 }
1314 #endif
1315 const Type* t = toop->filter_speculative(_type);
1316 if (t->singleton()) {
1317 // Don't replace InlineType by a constant
1318 t = _type;
1319 }
1320 const Type* tinit = phase->type(in(IsInit));
1321 if (tinit == Type::TOP) {
1322 return Type::TOP;
1323 }
1324 if (tinit->isa_int() && tinit->is_int()->is_con(1)) {
1325 t = t->join_speculative(TypePtr::NOTNULL);
1326 }
1327 return t;
1328 }