1 /*
2 * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "ci/ciInlineKlass.hpp"
26 #include "gc/shared/barrierSet.hpp"
27 #include "gc/shared/c2/barrierSetC2.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "memory/resourceArea.hpp"
30 #include "oops/accessDecorators.hpp"
31 #include "opto/addnode.hpp"
32 #include "opto/castnode.hpp"
33 #include "opto/cfgnode.hpp"
34 #include "opto/compile.hpp"
35 #include "opto/convertnode.hpp"
36 #include "opto/graphKit.hpp"
37 #include "opto/inlinetypenode.hpp"
38 #include "opto/memnode.hpp"
39 #include "opto/movenode.hpp"
40 #include "opto/multnode.hpp"
41 #include "opto/narrowptrnode.hpp"
42 #include "opto/node.hpp"
43 #include "opto/opcodes.hpp"
44 #include "opto/phaseX.hpp"
45 #include "opto/rootnode.hpp"
46 #include "opto/subnode.hpp"
47 #include "opto/type.hpp"
48 #include "utilities/globalDefinitions.hpp"
49 #include "utilities/growableArray.hpp"
50 #include "utilities/pair.hpp"
51 #include "utilities/tuple.hpp"
52
53 // Clones the inline type to handle control flow merges involving multiple inline types.
54 // The inputs are replaced by PhiNodes to represent the merged values for the given region.
55 // init_with_top: input of phis above the returned InlineTypeNode are initialized to top.
56 InlineTypeNode* InlineTypeNode::clone_with_phis(PhaseGVN* gvn, Node* region, SafePointNode* map, bool is_non_null, bool init_with_top) {
57 InlineTypeNode* vt = clone_if_required(gvn, map);
58 const Type* t = Type::get_const_type(inline_klass());
59 gvn->set_type(vt, t);
60 vt->as_InlineType()->set_type(t);
61
62 Node* const top = gvn->C->top();
63
64 // Create a PhiNode for merging the oop values
65 PhiNode* oop = PhiNode::make(region, init_with_top ? top : vt->get_oop(), t);
66 gvn->set_type(oop, t);
67 gvn->record_for_igvn(oop);
68 vt->set_oop(*gvn, oop);
69
70 // Create a PhiNode for merging the is_buffered values
71 t = Type::get_const_basic_type(T_BOOLEAN);
72 Node* is_buffered_node = PhiNode::make(region, init_with_top ? top : vt->get_is_buffered(), t);
73 gvn->set_type(is_buffered_node, t);
74 gvn->record_for_igvn(is_buffered_node);
75 vt->set_req(IsBuffered, is_buffered_node);
76
77 // Create a PhiNode for merging the null_marker values
78 Node* null_marker_node;
79 if (is_non_null) {
80 null_marker_node = gvn->intcon(1);
81 } else {
82 t = Type::get_const_basic_type(T_BOOLEAN);
83 null_marker_node = PhiNode::make(region, init_with_top ? top : vt->get_null_marker(), t);
84 gvn->set_type(null_marker_node, t);
85 gvn->record_for_igvn(null_marker_node);
86 }
87 vt->set_req(NullMarker, null_marker_node);
88
89 // Create a PhiNode each for merging the field values
90 for (uint i = 0; i < vt->field_count(); ++i) {
91 ciType* type = vt->field(i)->type();
92 Node* value = vt->field_value(i);
93 // We limit scalarization for inline types with circular fields and can therefore observe nodes
94 // of the same type but with different scalarization depth during GVN. To avoid inconsistencies
95 // during merging, make sure that we only create Phis for fields that are guaranteed to be scalarized.
96 ciField* field = this->field(i);
97 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
98 bool no_circularity = !gvn->C->has_circular_inline_type() || field->is_flat();
99 if (type->is_inlinetype() && no_circularity) {
100 // Handle inline type fields recursively
101 value = value->as_InlineType()->clone_with_phis(gvn, region, map);
102 } else {
103 t = Type::get_const_type(type);
104 value = PhiNode::make(region, init_with_top ? top : value, t);
105 gvn->set_type(value, t);
106 gvn->record_for_igvn(value);
107 }
108 vt->set_field_value(i, value);
109 }
110 gvn->record_for_igvn(vt);
111 return vt;
112 }
113
114 // Checks if the inputs of the InlineTypeNode were replaced by PhiNodes
115 // for the given region (see InlineTypeNode::clone_with_phis).
116 bool InlineTypeNode::has_phi_inputs(Node* region) const {
117 // Check oop input
118 bool result = get_oop()->is_Phi() && get_oop()->as_Phi()->region() == region;
119 #ifdef ASSERT
120 if (result) {
121 // Check all field value inputs for consistency
122 for (uint i = 0; i < field_count(); ++i) {
123 Node* n = field_value(i);
124 if (n->is_InlineType()) {
125 assert(n->as_InlineType()->has_phi_inputs(region), "inconsistent phi inputs");
126 } else {
127 assert(n->is_Phi() && n->as_Phi()->region() == region, "inconsistent phi inputs");
128 }
129 }
130 }
131 #endif
132 return result;
133 }
134
135 // Merges 'this' with 'other' by updating the input PhiNodes added by 'clone_with_phis'
136 InlineTypeNode* InlineTypeNode::merge_with(PhaseGVN* gvn, const InlineTypeNode* other, int phi_index, bool transform) {
137 assert(inline_klass() == other->inline_klass(), "Merging incompatible types");
138
139 // Merge oop inputs
140 PhiNode* phi = get_oop()->as_Phi();
141 phi->set_req(phi_index, other->get_oop());
142 if (transform) {
143 set_oop(*gvn, gvn->transform(phi));
144 }
145
146 // Merge is_buffered inputs
147 phi = get_is_buffered()->as_Phi();
148 phi->set_req(phi_index, other->get_is_buffered());
149 if (transform) {
150 set_req(IsBuffered, gvn->transform(phi));
151 }
152
153 // Merge null_marker inputs
154 Node* null_marker = get_null_marker();
155 if (null_marker->is_Phi()) {
156 phi = null_marker->as_Phi();
157 phi->set_req(phi_index, other->get_null_marker());
158 if (transform) {
159 set_req(NullMarker, gvn->transform(phi));
160 }
161 } else {
162 assert(null_marker->find_int_con(0) == 1, "only with a non null inline type");
163 }
164
165 // Merge field values
166 for (uint i = 0; i < field_count(); ++i) {
167 Node* val1 = field_value(i);
168 Node* val2 = other->field_value(i);
169 if (val1->is_InlineType()) {
170 if (val2->is_Phi()) {
171 val2 = gvn->transform(val2);
172 }
173 if (val2->is_top()) {
174 // The path where 'other' is used is dying. Therefore, we do not need to process the merge with 'other' further.
175 // The phi inputs of 'this' at 'phi_index' will eventually be removed.
176 break;
177 }
178 val1->as_InlineType()->merge_with(gvn, val2->as_InlineType(), phi_index, transform);
179 } else {
180 assert(val1->is_Phi(), "must be a phi node");
181 val1->set_req(phi_index, val2);
182 }
183 if (transform) {
184 set_field_value(i, gvn->transform(val1));
185 }
186 }
187 return this;
188 }
189
190 // Adds a new merge path to an inline type node with phi inputs
191 void InlineTypeNode::add_new_path(Node* region) const {
192 assert(has_phi_inputs(region), "must have phi inputs");
193
194 PhiNode* phi = get_oop()->as_Phi();
195 phi->add_req(nullptr);
196 assert(phi->req() == region->req(), "must be same size as region");
197
198 phi = get_is_buffered()->as_Phi();
199 phi->add_req(nullptr);
200 assert(phi->req() == region->req(), "must be same size as region");
201
202 phi = get_null_marker()->as_Phi();
203 phi->add_req(nullptr);
204 assert(phi->req() == region->req(), "must be same size as region");
205
206 for (uint i = 0; i < field_count(); ++i) {
207 Node* val = field_value(i);
208 if (val->is_InlineType()) {
209 val->as_InlineType()->add_new_path(region);
210 } else {
211 val->as_Phi()->add_req(nullptr);
212 assert(val->req() == region->req(), "must be same size as region");
213 }
214 }
215 }
216
217 Node* InlineTypeNode::field_value(uint index) const {
218 assert(index < field_count(), "index out of bounds");
219 return in(Values + index);
220 }
221
222 // Get the value of the field at the given offset.
223 // If 'recursive' is true, flat inline type fields will be resolved recursively.
224 Node* InlineTypeNode::field_value_by_offset(int offset, bool recursive) const {
225 // Find the declared field which contains the field we are looking for
226 int index = inline_klass()->field_index_by_offset(offset);
227 Node* value = field_value(index);
228 assert(value != nullptr, "field value not found");
229 ciField* field = this->field(index);
230 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
231
232 if (value->is_top()) {
233 // The graph is dying but a load may still ask for a nested field
234 // inside a flattened field before the dead load itself is folded away.
235 assert(offset == field->offset_in_bytes() || field->is_flat(), "offset mismatch");
236 return value;
237 }
238 if (!recursive || !field->is_flat()) {
239 assert(offset == field->offset_in_bytes(), "offset mismatch");
240 return value;
241 }
242
243 // Flat inline type field
244 InlineTypeNode* vt = value->as_InlineType();
245 assert(field->is_flat(), "must be flat");
246 if (offset == field->null_marker_offset()) {
247 return vt->get_null_marker();
248 } else {
249 int sub_offset = offset - field->offset_in_bytes(); // Offset of the flattened field inside the declared field
250 sub_offset += vt->inline_klass()->payload_offset(); // Add header size
251 return vt->field_value_by_offset(sub_offset, recursive);
252 }
253 }
254
255 void InlineTypeNode::set_field_value(uint index, Node* value) {
256 assert(index < field_count(), "index out of bounds");
257 set_req(Values + index, value);
258 }
259
260 void InlineTypeNode::set_field_value_by_offset(int offset, Node* value) {
261 set_field_value(field_index(offset), value);
262 }
263
264 uint InlineTypeNode::field_index(int offset) const {
265 uint i = 0;
266 for (; i < field_count() && field(i)->offset_in_bytes() != offset; i++) { }
267 assert(i < field_count(), "field not found");
268 return i;
269 }
270
271 ciField* InlineTypeNode::field(uint index) const {
272 assert(index < field_count(), "index out of bounds");
273 return inline_klass()->declared_nonstatic_field_at(index);
274 }
275
276 uint InlineTypeNode::add_fields_to_safepoint(Unique_Node_List& worklist, SafePointNode* sfpt) const {
277 uint cnt = 0;
278 for (uint i = 0; i < field_count(); ++i) {
279 Node* value = field_value(i);
280 ciField* field = this->field(i);
281 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
282 if (field->is_flat()) {
283 InlineTypeNode* vt = value->as_InlineType();
284 cnt += vt->add_fields_to_safepoint(worklist, sfpt);
285 if (!field->is_null_free()) {
286 // The null marker of a flat field is added right after we scalarize that field
287 sfpt->add_req(vt->get_null_marker());
288 cnt++;
289 }
290 continue;
291 }
292 if (value->is_InlineType()) {
293 // Add inline type to the worklist to process later
294 worklist.push(value);
295 }
296 sfpt->add_req(value);
297 cnt++;
298 }
299 return cnt;
300 }
301
302 void InlineTypeNode::make_scalar_in_safepoint(PhaseIterGVN* igvn, Unique_Node_List& worklist, SafePointNode* sfpt) const {
303 JVMState* jvms = sfpt->jvms();
304 assert(jvms != nullptr, "missing JVMS");
305 uint first_ind = (sfpt->req() - jvms->scloff());
306
307 // Iterate over the inline type fields in order of increasing offset and add the
308 // field values to the safepoint. Nullable inline types have a null marker field that
309 // needs to be checked before using the field values.
310 sfpt->add_req(get_null_marker());
311 uint nfields = add_fields_to_safepoint(worklist, sfpt);
312 jvms->set_endoff(sfpt->req());
313 // Replace safepoint edge by SafePointScalarObjectNode
314 SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(type()->isa_instptr(),
315 nullptr,
316 first_ind,
317 sfpt->jvms()->depth(),
318 nfields);
319 sobj->init_req(0, igvn->C->root());
320 sobj = igvn->transform(sobj)->as_SafePointScalarObject();
321 igvn->rehash_node_delayed(sfpt);
322 for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) {
323 Node* debug = sfpt->in(i);
324 if (debug != nullptr && debug->uncast() == this) {
325 sfpt->set_req(i, sobj);
326 }
327 }
328 }
329
330 void InlineTypeNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop) {
331 // If the inline type has a constant or loaded oop, use the oop instead of scalarization
332 // in the safepoint to avoid keeping field loads live just for the debug info.
333 Node* oop = get_oop();
334 bool use_oop = false;
335 if (allow_oop && is_allocated(igvn) && oop->is_Phi()) {
336 Unique_Node_List worklist;
337 VectorSet visited;
338 visited.set(oop->_idx);
339 worklist.push(oop);
340 use_oop = true;
341 while (worklist.size() > 0 && use_oop) {
342 Node* n = worklist.pop();
343 for (uint i = 1; i < n->req(); i++) {
344 Node* in = n->in(i);
345 if (in->is_Phi() && !visited.test_set(in->_idx)) {
346 worklist.push(in);
347 } else if (!(in->is_Con() || in->is_Parm())) {
348 use_oop = false;
349 break;
350 }
351 }
352 }
353 } else {
354 use_oop = allow_oop && is_allocated(igvn) &&
355 (oop->is_Con() || oop->is_Parm() || oop->is_Load() || (oop->isa_DecodeN() && oop->in(1)->is_Load()));
356 }
357
358 ResourceMark rm;
359 Unique_Node_List safepoints;
360 Unique_Node_List vt_worklist;
361 Unique_Node_List worklist;
362 worklist.push(this);
363 while (worklist.size() > 0) {
364 Node* n = worklist.pop();
365 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
366 Node* use = n->fast_out(i);
367 if (use->is_SafePoint() && !use->is_CallLeaf() && (!use->is_Call() || use->as_Call()->has_debug_use(n))) {
368 safepoints.push(use);
369 } else if (use->is_ConstraintCast()) {
370 worklist.push(use);
371 }
372 }
373 }
374
375 // Process all safepoint uses and scalarize inline type
376 while (safepoints.size() > 0) {
377 SafePointNode* sfpt = safepoints.pop()->as_SafePoint();
378 if (use_oop) {
379 for (uint i = sfpt->jvms()->debug_start(); i < sfpt->jvms()->debug_end(); i++) {
380 Node* debug = sfpt->in(i);
381 if (debug != nullptr && debug->uncast() == this) {
382 sfpt->set_req(i, get_oop());
383 }
384 }
385 igvn->rehash_node_delayed(sfpt);
386 } else {
387 make_scalar_in_safepoint(igvn, vt_worklist, sfpt);
388 }
389 }
390 // Now scalarize non-flat fields
391 for (uint i = 0; i < vt_worklist.size(); ++i) {
392 InlineTypeNode* vt = vt_worklist.at(i)->isa_InlineType();
393 vt->make_scalar_in_safepoints(igvn);
394 }
395 if (outcnt() == 0) {
396 igvn->record_for_igvn(this);
397 }
398 }
399
400 // We limit scalarization for inline types with circular fields and can therefore observe nodes
401 // of the same type but with different scalarization depth during GVN. This method adjusts the
402 // scalarization depth to avoid inconsistencies during merging.
403 InlineTypeNode* InlineTypeNode::adjust_scalarization_depth(GraphKit* kit) {
404 if (!kit->C->has_circular_inline_type()) {
405 return this;
406 }
407 GrowableArray<ciType*> visited;
408 visited.push(inline_klass());
409 return adjust_scalarization_depth_impl(kit, visited);
410 }
411
412 InlineTypeNode* InlineTypeNode::adjust_scalarization_depth_impl(GraphKit* kit, GrowableArray<ciType*>& visited) {
413 InlineTypeNode* val = this;
414 for (uint i = 0; i < field_count(); ++i) {
415 Node* value = field_value(i);
416 Node* new_value = value;
417 ciField* field = this->field(i);
418 ciType* ft = field->type();
419 if (value->is_InlineType()) {
420 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
421 if (!field->is_flat() && visited.contains(ft)) {
422 new_value = value->as_InlineType()->buffer(kit)->get_oop();
423 } else {
424 int old_len = visited.length();
425 visited.push(ft);
426 new_value = value->as_InlineType()->adjust_scalarization_depth_impl(kit, visited);
427 visited.trunc_to(old_len);
428 }
429 } else if (ft->is_inlinetype() && !visited.contains(ft)) {
430 int old_len = visited.length();
431 visited.push(ft);
432 new_value = make_from_oop_impl(kit, value, ft->as_inline_klass(), visited);
433 visited.trunc_to(old_len);
434 }
435 if (value != new_value) {
436 if (val == this) {
437 val = clone_if_required(&kit->gvn(), kit->map());
438 }
439 val->set_field_value(i, new_value);
440 }
441 }
442 return (val == this) ? this : kit->gvn().transform(val)->as_InlineType();
443 }
444
445 void InlineTypeNode::load(GraphKit* kit, Node* base, Node* ptr, bool immutable_memory, bool trust_null_free_oop, DecoratorSet decorators, GrowableArray<ciType*>& visited) {
446 // Initialize the inline type by loading its field values from
447 // memory and adding the values as input edges to the node.
448 ciInlineKlass* vk = inline_klass();
449 for (uint i = 0; i < field_count(); ++i) {
450 ciField* field = this->field(i);
451 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
452 int field_off = field->offset_in_bytes() - vk->payload_offset();
453 Node* field_ptr = kit->basic_plus_adr(base, ptr, field_off);
454 Node* value = nullptr;
455 ciType* ft = field->type();
456 bool field_null_free = field->is_null_free();
457 if (field->is_flat()) {
458 // Recursively load the flat inline type field
459 ciInlineKlass* fvk = ft->as_inline_klass();
460 bool atomic = field->is_atomic();
461
462 int old_len = visited.length();
463 visited.push(ft);
464 value = make_from_flat_impl(kit, fvk, base, field_ptr, atomic, immutable_memory,
465 field_null_free, trust_null_free_oop && field_null_free, decorators, visited);
466 visited.trunc_to(old_len);
467 } else {
468 // Load field value from memory
469 BasicType bt = type2field[ft->basic_type()];
470 assert(is_java_primitive(bt) || field_ptr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
471 const Type* val_type = Type::get_const_type(ft);
472 if (trust_null_free_oop && field_null_free) {
473 val_type = val_type->join_speculative(TypePtr::NOTNULL);
474 }
475 const TypePtr* field_ptr_type = (decorators & C2_MISMATCHED) == 0 ? kit->gvn().type(field_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
476 value = kit->access_load_at(base, field_ptr, field_ptr_type, val_type, bt, decorators);
477 // Loading a non-flattened inline type from memory
478 if (visited.contains(ft)) {
479 kit->C->set_has_circular_inline_type(true);
480 } else if (ft->is_inlinetype()) {
481 int old_len = visited.length();
482 visited.push(ft);
483 value = make_from_oop_impl(kit, value, ft->as_inline_klass(), visited);
484 visited.trunc_to(old_len);
485 }
486 }
487 set_field_value(i, value);
488 }
489 }
490
491 void InlineTypeNode::store_flat(GraphKit* kit, Node* base, Node* ptr, bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators) {
492 ciInlineKlass* vk = inline_klass();
493 bool do_atomic = atomic;
494 // With immutable memory, a non-atomic load and an atomic load are the same
495 if (immutable_memory) {
496 do_atomic = false;
497 }
498 // If there is only one flattened field, a non-atomic load and an atomic load are the same
499 if (vk->is_naturally_atomic(null_free)) {
500 do_atomic = false;
501 }
502
503 if (!do_atomic) {
504 if (!null_free) {
505 int nm_offset = vk->null_marker_offset_in_payload();
506 Node* nm_ptr = kit->basic_plus_adr(base, ptr, nm_offset);
507 const TypePtr* nm_ptr_type = (decorators & C2_MISMATCHED) == 0 ? kit->gvn().type(nm_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
508 kit->access_store_at(base, nm_ptr, nm_ptr_type, get_null_marker(), TypeInt::BOOL, T_BOOLEAN, decorators);
509 }
510 store(kit, base, ptr, immutable_memory, decorators);
511 return;
512 }
513
514 StoreFlatNode::store(kit, base, ptr, this, null_free, decorators);
515 }
516
517 void InlineTypeNode::store_flat_array(GraphKit* kit, Node* base, Node* idx) {
518 PhaseGVN& gvn = kit->gvn();
519 DecoratorSet decorators = IN_HEAP | IS_ARRAY | MO_UNORDERED;
520 kit->C->set_flat_accesses();
521 ciInlineKlass* vk = inline_klass();
522 assert(vk->maybe_flat_in_array(), "element type %s cannot be flat in array", vk->name()->as_utf8());
523
524 RegionNode* region = new RegionNode(4);
525 gvn.set_type(region, Type::CONTROL);
526 kit->record_for_igvn(region);
527
528 Node* input_memory_state = kit->reset_memory();
529 kit->set_all_memory(input_memory_state);
530
531 PhiNode* mem = PhiNode::make(region, input_memory_state, Type::MEMORY, TypePtr::BOTTOM);
532 gvn.set_type(mem, Type::MEMORY);
533 kit->record_for_igvn(mem);
534
535 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO);
536 gvn.set_type(io, Type::ABIO);
537 kit->record_for_igvn(io);
538
539 Node* bol_null_free = kit->null_free_array_test(base); // Argument evaluation order is undefined in C++ and since this sets control, it needs to come first
540 IfNode* iff_null_free = kit->create_and_map_if(kit->control(), bol_null_free, PROB_FAIR, COUNT_UNKNOWN);
541
542 // Nullable
543 kit->set_control(kit->IfFalse(iff_null_free));
544 if (!kit->stopped()) {
545 assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8());
546 kit->set_all_memory(input_memory_state);
547 Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true);
548 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
549 store_flat(kit, cast, ptr, true, false, false, decorators);
550
551 region->init_req(1, kit->control());
552 mem->set_req(1, kit->reset_memory());
553 io->set_req(1, kit->i_o());
554 }
555
556 // Null-free
557 kit->set_control(kit->IfTrue(iff_null_free));
558 if (!kit->stopped()) {
559 kit->set_all_memory(input_memory_state);
560
561 Node* bol_atomic = kit->null_free_atomic_array_test(base, vk);
562 IfNode* iff_atomic = kit->create_and_map_if(kit->control(), bol_atomic, PROB_FAIR, COUNT_UNKNOWN);
563
564 // Atomic
565 kit->set_control(kit->IfTrue(iff_atomic));
566 if (!kit->stopped()) {
567 assert(vk->has_null_free_atomic_layout(), "element type %s does not have a null-free atomic flat layout", vk->name()->as_utf8());
568 kit->set_all_memory(input_memory_state);
569 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, true);
570 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
571 store_flat(kit, cast, ptr, true, false, true, decorators);
572
573 region->init_req(2, kit->control());
574 mem->set_req(2, kit->reset_memory());
575 io->set_req(2, kit->i_o());
576 }
577
578 // Non-atomic
579 kit->set_control(kit->IfFalse(iff_atomic));
580 if (!kit->stopped()) {
581 assert(vk->has_null_free_non_atomic_layout(), "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8());
582 kit->set_all_memory(input_memory_state);
583 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, false);
584 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
585 store_flat(kit, cast, ptr, false, false, true, decorators);
586
587 region->init_req(3, kit->control());
588 mem->set_req(3, kit->reset_memory());
589 io->set_req(3, kit->i_o());
590 }
591 }
592
593 kit->set_control(gvn.transform(region));
594 kit->set_all_memory(gvn.transform(mem));
595 kit->set_i_o(gvn.transform(io));
596 }
597
598 void InlineTypeNode::store(GraphKit* kit, Node* base, Node* ptr, bool immutable_memory, DecoratorSet decorators) const {
599 // Write field values to memory
600 ciInlineKlass* vk = inline_klass();
601 for (uint i = 0; i < field_count(); ++i) {
602 ciField* field = this->field(i);
603 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
604 int field_off = field->offset_in_bytes() - vk->payload_offset();
605 Node* field_val = field_value(i);
606 bool field_null_free = field->is_null_free();
607 ciType* ft = field->type();
608 Node* field_ptr = kit->basic_plus_adr(base, ptr, field_off);
609 if (field->is_flat()) {
610 // Recursively store the flat inline type field
611 bool atomic = field->is_atomic();
612 field_val->as_InlineType()->store_flat(kit, base, field_ptr, atomic, immutable_memory, field_null_free, decorators);
613 } else {
614 // Store field value to memory
615 BasicType bt = type2field[ft->basic_type()];
616 const TypePtr* field_ptr_type = (decorators & C2_MISMATCHED) == 0 ? kit->gvn().type(field_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
617 const Type* val_type = Type::get_const_type(ft);
618 kit->access_store_at(base, field_ptr, field_ptr_type, field_val, val_type, bt, decorators);
619 }
620 }
621 }
622
623 // If a value class contains cycle, bail out from trying to expand substitutability check involving it
624 static bool check_cycle(ciInlineKlass* vk) {
625 ResourceMark rm;
626 GrowableArray<Pair<ciInlineKlass*, int>> visited;
627 visited.push(Pair(vk, 0));
628 while (visited.is_nonempty()) {
629 ciInlineKlass* current = visited.top().first;
630 bool finish = true;
631 for (int field_idx = visited.top().second; field_idx < current->nof_nonstatic_fields(); field_idx++) {
632 ciField* field = current->nonstatic_field_at(field_idx);
633 ciType* ft = field->type();
634 if (!ft->is_inlinetype()) {
635 continue;
636 } else if (visited.find_if([&](const auto& entry) { return entry.first == ft; }) >= 0) {
637 return true;
638 } else {
639 visited.top().second = field_idx + 1;
640 visited.push(Pair(ft->as_inline_klass(), 0));
641 finish = false;
642 break;
643 }
644 }
645 if (finish) {
646 visited.pop();
647 }
648 }
649 return false;
650 }
651
652 // Check if a substitutability check between 'lhs' and 'rhs' can be implemented in IR
653 bool InlineTypeNode::can_emit_substitutability_check(Node* lhs, Node* rhs) {
654 if (!lhs->bottom_type()->isa_ptr() ||
655 (rhs != nullptr && !rhs->bottom_type()->isa_ptr())) {
656 return false;
657 }
658
659 if (rhs != nullptr && lhs->eqv_uncast(rhs)) {
660 return true;
661 }
662
663 if (!lhs->bottom_type()->is_ptr()->can_be_inline_type() ||
664 (rhs != nullptr && !rhs->bottom_type()->is_ptr()->can_be_inline_type())) {
665 return true;
666 }
667
668 if (!lhs->is_InlineType() && (rhs == nullptr || !rhs->is_InlineType())) {
669 return false;
670 }
671
672 if (!lhs->is_InlineType()) {
673 swap(lhs, rhs);
674 }
675
676 InlineTypeNode* lhs_inline = lhs->as_InlineType();
677 InlineTypeNode* rhs_inline = rhs != nullptr ? rhs->isa_InlineType() : nullptr;
678 if (rhs_inline != nullptr && lhs_inline->type()->inline_klass() != rhs_inline->type()->inline_klass()) {
679 // Dead code, can skip the substitutability check
680 return true;
681 }
682
683 if (check_cycle(lhs_inline->inline_klass())) {
684 return false;
685 }
686
687 for (uint i = 0; i < lhs_inline->field_count(); i++) {
688 ciType* ft = lhs_inline->field(i)->type();
689 if (!ft->can_be_inline_klass()) {
690 continue;
691 }
692
693 Node* lhs_fv = lhs_inline->field_value(i);
694 Node* rhs_fv = rhs_inline != nullptr ? rhs_inline->field_value(i) : nullptr;
695 if (!can_emit_substitutability_check(lhs_fv, rhs_fv)) {
696 return false;
697 }
698 }
699 return true;
700 }
701
702 // Compare lhs and rhs given they are not value objects
703 static void emit_substitutability_check_primitive(GraphKit* kit, PhiNode* result, Node* lhs, Node* rhs, BasicType bt) {
704 PhaseGVN& gvn = kit->gvn();
705 Node* cmp;
706 if (bt == T_INT || is_subword_type(bt)) {
707 cmp = kit->CmpI(lhs, rhs);
708 } else if (bt == T_LONG) {
709 cmp = kit->CmpL(lhs, rhs);
710 } else if (bt == T_FLOAT) {
711 lhs = gvn.transform(new MoveF2INode(lhs));
712 rhs = gvn.transform(new MoveF2INode(rhs));
713 cmp = kit->CmpI(lhs, rhs);
714 } else if (bt == T_DOUBLE) {
715 lhs = gvn.transform(new MoveD2LNode(lhs));
716 rhs = gvn.transform(new MoveD2LNode(rhs));
717 cmp = kit->CmpL(lhs, rhs);
718 } else {
719 assert(is_reference_type(bt), "unexpected bt %s", type2name(bt));
720 cmp = kit->CmpP(lhs, rhs);
721 }
722
723 Node* bol = kit->Bool(cmp, BoolTest::eq);
724 IfNode* iff = kit->create_and_map_if(kit->control(), bol, PROB_FAIR, COUNT_UNKNOWN);
725
726 Node* iff_false = kit->IfFalse(iff);
727 result->add_req(kit->intcon(0));
728 result->region()->add_req(iff_false);
729
730 Node* iff_true = kit->IfTrue(iff);
731 kit->set_control(iff_true);
732 }
733
734 // Try to see what should be done with lhs and rhs. Either we can emit the answer if it is simple,
735 // give up and emit a call to runtime, or start comparing the value objects field-by-field. In the
736 // last case, NodeSentinel is returned.
737 static Node* emit_substitutability_check_pointer(GraphKit* kit, PhiNode* result, Node* lhs, Node* rhs) {
738 PhaseGVN& gvn = kit->gvn();
739 Node* top = kit->C->top();
740 const Type* lhs_type = gvn.type(lhs);
741 const Type* rhs_type = gvn.type(rhs);
742
743 // Dead graph
744 if (lhs_type == Type::TOP || rhs_type == Type::TOP) {
745 kit->set_control(top);
746 return top;
747 }
748
749 Node* cmp = nullptr;
750 if (lhs->eqv_uncast(rhs)) {
751 cmp = kit->intcon(0);
752 } else if (!lhs_type->is_ptr()->can_be_inline_type() || !rhs_type->is_ptr()->can_be_inline_type()) {
753 // If one of the sides is not a value object, can only be substitutable if they are the same
754 cmp = kit->CmpP(lhs, rhs);
755 } else if (lhs_type->is_instptr()->as_klass_type()->join(rhs_type->is_instptr()->as_klass_type())->empty()) {
756 // Both belongs to provably different types, must be different unless both are null. Cannot
757 // rely on the pointer independence alone because different pointers may still be
758 // substitutable.
759 cmp = kit->CmpP(lhs, rhs);
760 }
761 if (cmp != nullptr) {
762 Node* res = kit->Bool(cmp, BoolTest::eq);
763 IfNode* iff = kit->create_and_map_if(kit->control(), res, PROB_FAIR, COUNT_UNKNOWN);
764
765 Node* iff_false = kit->IfFalse(iff);
766 result->region()->add_req(iff_false);
767 result->add_req(gvn.intcon(0));
768
769 Node* iff_true = kit->IfTrue(iff);
770 kit->set_control(iff_true);
771 return iff_true;
772 }
773
774 if (!lhs_type->is_inlinetypeptr() && !rhs_type->is_inlinetypeptr()) {
775 return nullptr;
776 }
777
778 // A cycle, give up
779 ciInlineKlass* vk = lhs_type->is_inlinetypeptr() ? lhs_type->inline_klass() : rhs_type->inline_klass();
780 if (check_cycle(vk)) {
781 return nullptr;
782 }
783
784 return NodeSentinel;
785 }
786
787 Node* InlineTypeNode::emit_substitutability_check(GraphKit* kit, Node* lhs, Node* rhs) {
788 if (!kit->C->allow_macro_nodes()) {
789 // After macro expansion, InlineTypeNodes are also eliminated, creation of new ones then is not
790 // allowed
791 return nullptr;
792 }
793
794 PhaseIterGVN& igvn = *kit->gvn().is_IterGVN();
795 RegionNode* region = new RegionNode(1);
796 PhiNode* result = new PhiNode(region, TypeInt::BOOL);
797 igvn.register_new_node_with_optimizer(region);
798 igvn.register_new_node_with_optimizer(result);
799
800 Node* preprocess = emit_substitutability_check_pointer(kit, result, lhs, rhs);
801 if (preprocess == nullptr) {
802 return nullptr;
803 } else if (preprocess != NodeSentinel) {
804 region->add_req(kit->control());
805 result->add_req(igvn.intcon(1));
806 kit->set_control(region);
807 return result;
808 }
809
810 const Type* lhs_type = igvn.type(lhs);
811 const Type* rhs_type = igvn.type(rhs);
812 assert(!lhs_type->maybe_null() && !rhs_type->maybe_null(), "must check null beforehand");
813 ciInlineKlass* vk = lhs_type->is_inlinetypeptr() ? lhs_type->inline_klass() : rhs_type->inline_klass();
814 const TypeInstPtr* vk_type = TypeOopPtr::make_from_klass(vk)->join(TypePtr::NOTNULL)->is_instptr();
815
816 if (!lhs->is_InlineType()) {
817 if (!lhs_type->higher_equal(vk_type)) {
818 lhs = igvn.transform(new CheckCastPPNode(kit->control(), lhs, vk_type, ConstraintCastNode::DependencyType::NonFloatingNarrowing));
819 }
820 lhs = InlineTypeNode::make_from_oop(kit, lhs, vk);
821 }
822 if (!rhs->is_InlineType()) {
823 if (!rhs_type->higher_equal(vk_type)) {
824 rhs = igvn.transform(new CheckCastPPNode(kit->control(), rhs, vk_type, ConstraintCastNode::DependencyType::NonFloatingNarrowing));
825 }
826 rhs = InlineTypeNode::make_from_oop(kit, rhs, vk);
827 }
828
829 RegionNode* true_region = new RegionNode(3);
830 igvn.register_new_node_with_optimizer(true_region);
831 region->add_req(true_region);
832 result->add_req(igvn.intcon(1));
833
834 // To verify the substitutability of 2 notnull value objects of the same type, we need to check
835 // the substitutability of each field in the objects:
836 //
837 // substitutable(cur_lhs, cur_rhs) {
838 // cur_start_region:
839 //
840 // if (cur_lhs.field1 == null && cur_rhs.field1 == null) {
841 // goto cur_true_field1_region;
842 // } else if ((cur_lhs.field1 == null) != (cur_rhs.field1 == null) {
843 // return false;
844 // } else if (cur_lhs.field1.klass != cur_rhs.field1.klass) {
845 // return false;
846 // }
847 //
848 // cur_start_field1_region:
849 // if (substitutable(cur_lhs.field1, cur_rhs.field1)) {
850 // return false;
851 // }
852 // cur_true_field1_region:
853 //
854 // if (cur_lhs.field2 == null && cur_rhs.field2 == null) {
855 // goto cur_true_field2_region;
856 // } else if ((cur_lhs.field2 == null) != (cur_rhs.field2 == null) {
857 // return false;
858 // } else if (cur_lhs.field2.klass != cur_rhs.field2.klass) {
859 // return false;
860 // }
861 //
862 // cur_start_field2_region:
863 // if (!substitutable(cur_lhs.field1, cur_rhs.field1)) {
864 // return false;
865 // }
866 // cur_true_field2_region:
867 //
868 // ...
869 //
870 // cur_true_fieldn_region:
871 //
872 // cur_true_region:
873 // }
874 //
875 // To avoid recursion, for field values that are value objects, we create the start and end
876 // region for the substitutability check of that field when both sides are notnull, save them on
877 // the worklist to process them later.
878 ResourceMark rm;
879 using WorklistEntry = Tuple<Node*, RegionNode*, InlineTypeNode*, InlineTypeNode*>;
880 GrowableArray<WorklistEntry> worklist;
881 worklist.push(WorklistEntry(kit->control(), true_region, lhs->as_InlineType(), rhs->as_InlineType()));
882 while (worklist.is_nonempty()) {
883 WorklistEntry entry = worklist.pop();
884 Node* cur_start_region = entry.get<0>();
885 RegionNode* cur_true_region = entry.get<1>();
886 InlineTypeNode* cur_lhs = entry.get<2>();
887 InlineTypeNode* cur_rhs = entry.get<3>();
888
889 kit->set_control(cur_start_region);
890 if (kit->stopped()) {
891 break;
892 }
893 ciInlineKlass* vk = cur_lhs->inline_klass();
894 assert(vk == cur_rhs->inline_klass(), "should not reach here otherwise");
895
896 auto is_simple_field = [](ciField* field) {
897 ciType* ft = field->type();
898 return is_java_primitive(ft->basic_type()) || !ft->as_klass()->can_be_inline_klass();
899 };
900
901 // Go through all fields, process the simple ones first
902 for (int field_idx = 0; field_idx < vk->nof_declared_nonstatic_fields(); field_idx++) {
903 ciField* field = vk->declared_nonstatic_field_at(field_idx);
904 if (is_simple_field(field)) {
905 Node* cur_lhs_field = cur_lhs->field_value(field_idx);
906 Node* cur_rhs_field = cur_rhs->field_value(field_idx);
907 emit_substitutability_check_primitive(kit, result, cur_lhs_field, cur_rhs_field, field->type()->basic_type());
908 }
909 }
910
911 for (int field_idx = 0; field_idx < vk->nof_declared_nonstatic_fields(); field_idx++) {
912 ciField* field = vk->declared_nonstatic_field_at(field_idx);
913 if (is_simple_field(field)) {
914 // These fields have already been processed
915 continue;
916 }
917
918 Node* cur_lhs_field = cur_lhs->field_value(field_idx);
919 Node* cur_rhs_field = cur_rhs->field_value(field_idx);
920
921 Node* preprocess = emit_substitutability_check_pointer(kit, result, cur_lhs_field, cur_rhs_field);
922 if (kit->stopped()) {
923 break;
924 } else if (preprocess == nullptr) {
925 // Must emit a substitutability check for this field, give up for now
926 return nullptr;
927 } else if (preprocess != NodeSentinel) {
928 continue;
929 }
930
931 // When field is not null-free, the shape would look like this:
932 //
933 // current_control:
934 // if (cur_lhs_field == null) {
935 // if (cur_rhs_field == null) {
936 // goto field_true_region;
937 // } else {
938 // return false;
939 // }
940 // } else {
941 // if (cur_rhs_field == null) {
942 // return false;
943 // }
944 // }
945 // if (cur_lhs_field.klass != cur_rhs_field.klass) {
946 // return false;
947 // }
948 //
949 // field_start_region:
950 // if (!substitutable(cur_lhs_field, cur_rhs_field)) {
951 // return false;
952 // }
953 // field_true_region:
954 //
955 // The substitutability test between the fields of cur_lhs_field and cur_rhs_field is then
956 // pushed on the worklist to be expanded later.
957 RegionNode* field_true_region = new RegionNode(3);
958 igvn.register_new_node_with_optimizer(field_true_region);
959
960 // Firstly, filter the cases when either is null
961 Node* null_unknown = kit->top();
962 cur_lhs_field = kit->null_check_common(cur_lhs_field, T_OBJECT, false, &null_unknown);
963 if (!null_unknown->is_top()) {
964 PreserveJVMState pjvms(kit);
965 kit->set_control(null_unknown);
966 Node* null_null = kit->top();
967 kit->null_check_common(cur_rhs_field, T_OBJECT, false, &null_null);
968
969 // null - null, skip other checks
970 field_true_region->init_req(1, null_null);
971
972 // null - notnull
973 if (!kit->stopped()) {
974 region->add_req(kit->control());
975 result->add_req(igvn.intcon(0));
976 }
977 }
978 if (kit->stopped()) {
979 // cur_lhs_field is always null
980 kit->set_control(field_true_region);
981 continue;
982 }
983
984 Node* notnull_null = kit->top();
985 cur_rhs_field = kit->null_check_common(cur_rhs_field, T_OBJECT, false, ¬null_null);
986 if (!notnull_null->is_top()) {
987 region->add_req(notnull_null);
988 result->add_req(igvn.intcon(0));
989 }
990
991 if (kit->stopped()) {
992 // cur_rhs_field is always null
993 kit->set_control(field_true_region);
994 continue;
995 }
996 assert(!igvn.type(cur_lhs_field)->maybe_null() && !igvn.type(cur_rhs_field)->maybe_null(), "must be notnull");
997
998 // Can expand this comparison
999 const Type* cur_lhs_field_type = igvn.type(cur_lhs_field);
1000 const Type* cur_rhs_field_type = igvn.type(cur_rhs_field);
1001 ciInlineKlass* vk = cur_lhs_field_type->is_inlinetypeptr() ? cur_lhs_field_type->inline_klass() : cur_rhs_field_type->inline_klass();
1002 const TypeInstKlassPtr* vk_klass_type = TypeInstKlassPtr::make(vk, Type::ignore_interfaces);
1003 Node* vk_klass = igvn.makecon(vk_klass_type);
1004
1005 // Both must be vk
1006 Node* not_vk = kit->top();
1007 cur_lhs_field = kit->gen_checkcast(cur_lhs_field, vk_klass, ¬_vk);
1008 if (!not_vk->is_top()) {
1009 region->add_req(not_vk);
1010 result->add_req(igvn.intcon(0));
1011 }
1012 not_vk = kit->top();
1013 cur_rhs_field = kit->gen_checkcast(cur_rhs_field, vk_klass, ¬_vk);
1014 if (!not_vk->is_top()) {
1015 region->add_req(not_vk);
1016 result->add_req(igvn.intcon(0));
1017 }
1018 if (!kit->stopped()) {
1019 // Push the expanded InlineTypeNodes for processing later
1020 worklist.push(WorklistEntry(kit->control(), field_true_region, cur_lhs_field->as_InlineType(), cur_rhs_field->as_InlineType()));
1021 }
1022
1023 kit->set_control(field_true_region);
1024 }
1025
1026 cur_true_region->init_req(2, kit->control());
1027 kit->set_control(cur_true_region);
1028 }
1029
1030 kit->set_control(region);
1031 return result;
1032 }
1033
1034 InlineTypeNode* InlineTypeNode::buffer(GraphKit* kit, bool safe_for_replace) {
1035 if (is_allocated(&kit->gvn())) {
1036 // Already buffered
1037 return this;
1038 }
1039
1040 // Check if inline type is already buffered
1041 Node* not_buffered_ctl = kit->top();
1042 Node* not_null_oop = kit->null_check_oop(get_oop(), ¬_buffered_ctl, /* never_see_null = */ false, safe_for_replace);
1043 if (not_buffered_ctl->is_top()) {
1044 // Already buffered
1045 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map(), safe_for_replace);
1046 vt->set_is_buffered(kit->gvn());
1047 vt = kit->gvn().transform(vt)->as_InlineType();
1048 if (safe_for_replace) {
1049 kit->replace_in_map(this, vt);
1050 }
1051 return vt;
1052 }
1053 Node* buffered_ctl = kit->control();
1054 kit->set_control(not_buffered_ctl);
1055
1056 // Inline type is not buffered, check if it is null.
1057 Node* null_ctl = kit->top();
1058 kit->null_check_common(get_null_marker(), T_INT, false, &null_ctl);
1059 bool null_free = null_ctl->is_top();
1060
1061 RegionNode* region = new RegionNode(4);
1062 PhiNode* oop = PhiNode::make(region, not_null_oop, type()->join_speculative(null_free ? TypePtr::NOTNULL : TypePtr::BOTTOM));
1063
1064 // InlineType is already buffered
1065 region->init_req(1, buffered_ctl);
1066 oop->init_req(1, not_null_oop);
1067
1068 // InlineType is null
1069 region->init_req(2, null_ctl);
1070 oop->init_req(2, kit->gvn().zerocon(T_OBJECT));
1071
1072 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO);
1073 PhiNode* mem = PhiNode::make(region, kit->merged_memory(), Type::MEMORY, TypePtr::BOTTOM);
1074
1075 if (!kit->stopped()) {
1076 assert(!is_allocated(&kit->gvn()), "already buffered");
1077 PreserveJVMState pjvms(kit);
1078 ciInlineKlass* vk = inline_klass();
1079 // Allocate and initialize buffer, re-execute on deoptimization.
1080 kit->jvms()->set_bci(kit->bci());
1081 kit->jvms()->set_should_reexecute(true);
1082 kit->kill_dead_locals();
1083 Node* klass_node = kit->makecon(TypeKlassPtr::make(vk));
1084 Node* alloc_oop = kit->new_instance(klass_node, nullptr, nullptr, /* deoptimize_on_exception */ true, this);
1085 Node* payload_alloc_oop = kit->basic_plus_adr(alloc_oop, vk->payload_offset());
1086 store(kit, alloc_oop, payload_alloc_oop, true, IN_HEAP | MO_UNORDERED | C2_TIGHTLY_COUPLED_ALLOC);
1087
1088 // Do not let stores that initialize this buffer be reordered with a subsequent
1089 // store that would make this buffer accessible by other threads.
1090 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_oop);
1091 assert(alloc != nullptr, "must have an allocation node");
1092 kit->insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress));
1093 oop->init_req(3, alloc_oop);
1094 region->init_req(3, kit->control());
1095 io ->init_req(3, kit->i_o());
1096 mem ->init_req(3, kit->merged_memory());
1097 }
1098
1099 // Update GraphKit
1100 kit->set_control(kit->gvn().transform(region));
1101 kit->set_i_o(kit->gvn().transform(io));
1102 kit->set_all_memory(kit->gvn().transform(mem));
1103 kit->record_for_igvn(region);
1104 kit->record_for_igvn(oop);
1105 kit->record_for_igvn(io);
1106 kit->record_for_igvn(mem);
1107
1108 // Use cloned InlineTypeNode to propagate oop from now on
1109 Node* res_oop = kit->gvn().transform(oop);
1110 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map(), safe_for_replace);
1111 vt->set_oop(kit->gvn(), res_oop);
1112 vt->set_is_buffered(kit->gvn());
1113 vt = kit->gvn().transform(vt)->as_InlineType();
1114 kit->record_for_igvn(vt);
1115 if (safe_for_replace) {
1116 kit->replace_in_map(this, vt);
1117 }
1118 // InlineTypeNode::remove_redundant_allocations piggybacks on split if.
1119 // Make sure it gets a chance to remove this allocation.
1120 kit->C->set_has_split_ifs(true);
1121 return vt;
1122 }
1123
1124 bool InlineTypeNode::is_allocated(PhaseGVN* phase) const {
1125 if (phase->type(get_is_buffered()) == TypeInt::ONE) {
1126 return true;
1127 }
1128 Node* oop = get_oop();
1129 const Type* oop_type = (phase != nullptr) ? phase->type(oop) : oop->bottom_type();
1130 return !oop_type->maybe_null();
1131 }
1132
1133 static void replace_proj(Compile* C, CallNode* call, uint& proj_idx, Node* value, BasicType bt) {
1134 ProjNode* pn = call->proj_out_or_null(proj_idx);
1135 if (pn != nullptr) {
1136 C->gvn_replace_by(pn, value);
1137 C->initial_gvn()->hash_delete(pn);
1138 pn->set_req(0, C->top());
1139 }
1140 proj_idx += type2size[bt];
1141 }
1142
1143 // When a call returns multiple values, it has several result
1144 // projections, one per field. Replacing the result of the call by an
1145 // inline type node (after late inlining) requires that for each result
1146 // projection, we find the corresponding inline type field.
1147 void InlineTypeNode::replace_call_results(GraphKit* kit, CallNode* call, Compile* C) const {
1148 uint proj_idx = TypeFunc::Parms;
1149 // Replace oop projection
1150 replace_proj(C, call, proj_idx, get_oop(), T_OBJECT);
1151 // Replace field projections
1152 replace_field_projs(C, call, proj_idx);
1153 // Replace null_marker projection
1154 replace_proj(C, call, proj_idx, get_null_marker(), T_BOOLEAN);
1155 assert(proj_idx == call->tf()->range_cc()->cnt(), "missed a projection");
1156 }
1157
1158 void InlineTypeNode::replace_field_projs(Compile* C, CallNode* call, uint& proj_idx) const {
1159 for (uint i = 0; i < field_count(); ++i) {
1160 Node* value = field_value(i);
1161 ciField* field = this->field(i);
1162 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1163 if (field->is_flat()) {
1164 InlineTypeNode* vt = value->as_InlineType();
1165 // Replace field projections for flat field
1166 vt->replace_field_projs(C, call, proj_idx);
1167 if (!field->is_null_free()) {
1168 // Replace null_marker projection for nullable field
1169 replace_proj(C, call, proj_idx, vt->get_null_marker(), T_BOOLEAN);
1170 }
1171 continue;
1172 }
1173 // Replace projection for field value
1174 replace_proj(C, call, proj_idx, value, field->type()->basic_type());
1175 }
1176 }
1177
1178 InlineTypeNode* InlineTypeNode::allocate_fields(GraphKit* kit) {
1179 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map());
1180 for (uint i = 0; i < field_count(); i++) {
1181 Node* value = field_value(i);
1182 ciField* field = this->field(i);
1183 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1184 if (field->is_flat()) {
1185 // Flat inline type field
1186 vt->set_field_value(i, value->as_InlineType()->allocate_fields(kit));
1187 } else if (value->is_InlineType()) {
1188 // Non-flat inline type field
1189 vt->set_field_value(i, value->as_InlineType()->buffer(kit));
1190 }
1191 }
1192 vt = kit->gvn().transform(vt)->as_InlineType();
1193 kit->replace_in_map(this, vt);
1194 return vt;
1195 }
1196
1197 // Replace a buffer allocation by a dominating allocation
1198 static void replace_allocation(PhaseIterGVN* igvn, Node* res, Node* dom) {
1199 // Remove initializing stores and GC barriers
1200 for (DUIterator_Fast imax, i = res->fast_outs(imax); i < imax; i++) {
1201 Node* use = res->fast_out(i);
1202 if (use->is_AddP()) {
1203 for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) {
1204 Node* store = use->fast_out(j)->isa_Store();
1205 if (store != nullptr) {
1206 igvn->rehash_node_delayed(store);
1207 igvn->replace_in_uses(store, store->in(MemNode::Memory));
1208 }
1209 }
1210 } else if (use->Opcode() == Op_CastP2X) {
1211 if (UseG1GC && use->find_out_with(Op_XorX)->in(1) != use) {
1212 // The G1 pre-barrier uses a CastP2X both for the pointer of the object
1213 // we store into, as well as the value we are storing. Skip if this is a
1214 // barrier for storing 'res' into another object.
1215 continue;
1216 }
1217 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1218 bs->eliminate_gc_barrier(igvn, use);
1219 --i; --imax;
1220 }
1221 }
1222 igvn->replace_node(res, dom);
1223 }
1224
1225 Node* InlineTypeNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1226 Node* oop = get_oop();
1227 if (oop->isa_InlineType() && !phase->type(oop)->maybe_null()) {
1228 InlineTypeNode* vtptr = oop->as_InlineType();
1229 assert(inline_klass() == vtptr->inline_klass(), "inconsistent types");
1230 set_oop(*phase, vtptr->get_oop());
1231 set_is_buffered(*phase);
1232 set_null_marker(*phase);
1233 for (uint i = Values; i < vtptr->req(); ++i) {
1234 set_req(i, vtptr->in(i));
1235 }
1236 return this;
1237 }
1238
1239 // Use base oop if fields are loaded from memory, don't do so if base is the CheckCastPP of an
1240 // allocation because the only case we load from a naked CheckCastPP is when we exit a
1241 // constructor of an inline type and we want to relinquish the larval oop there. This has a
1242 // couple of benefits:
1243 // - The allocation is likely to be elided earlier if it is not an input of an InlineTypeNode.
1244 // - The InlineTypeNode without an allocation input is more likely to be GVN-ed. This may emerge
1245 // when we try to clone a value object.
1246 // - The buffering, if needed, is delayed until it is required. This new allocation, since it is
1247 // created from an InlineTypeNode, is recognized as not having a unique identity and in the
1248 // future, we can move them around more freely such as hoisting out of loops. This is not true
1249 // for the old allocation since larval value objects do have unique identities.
1250 Node* base = is_loaded(phase);
1251 if (base != nullptr && !base->is_InlineType() && !phase->type(base)->maybe_null() && phase->C->allow_macro_nodes() && AllocateNode::Ideal_allocation(base) == nullptr) {
1252 if (oop != base || !is_allocated(phase)) {
1253 set_oop(*phase, base);
1254 set_is_buffered(*phase);
1255 return this;
1256 }
1257 }
1258
1259 if (can_reshape) {
1260 PhaseIterGVN* igvn = phase->is_IterGVN();
1261 if (is_allocated(phase)) {
1262 // Search for and remove re-allocations of this inline type. Ignore scalar replaceable ones,
1263 // they will be removed anyway and changing the memory chain will confuse other optimizations.
1264 // This can happen with late inlining when we first allocate an inline type argument
1265 // but later decide to inline the call after the callee code also triggered allocation.
1266 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1267 AllocateNode* alloc = fast_out(i)->isa_Allocate();
1268 if (alloc != nullptr && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable) {
1269 // Found a re-allocation
1270 Node* res = alloc->result_cast();
1271 if (res != nullptr && res->is_CheckCastPP()) {
1272 // Replace allocation by oop and unlink AllocateNode
1273 replace_allocation(igvn, res, oop);
1274 igvn->replace_input_of(alloc, AllocateNode::InlineType, igvn->C->top());
1275 --i; --imax;
1276 }
1277 }
1278 }
1279 }
1280 }
1281
1282 return nullptr;
1283 }
1284
1285 InlineTypeNode* InlineTypeNode::make_uninitialized(PhaseGVN& gvn, ciInlineKlass* vk, bool null_free) {
1286 // Create a new InlineTypeNode with uninitialized values and nullptr oop
1287 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT), null_free);
1288 vt->set_is_buffered(gvn, false);
1289 vt->set_null_marker(gvn);
1290 return vt;
1291 }
1292
1293 InlineTypeNode* InlineTypeNode::make_all_zero(PhaseGVN& gvn, ciInlineKlass* vk) {
1294 GrowableArray<ciType*> visited;
1295 visited.push(vk);
1296 return make_all_zero_impl(gvn, vk, visited);
1297 }
1298
1299 InlineTypeNode* InlineTypeNode::make_all_zero_impl(PhaseGVN& gvn, ciInlineKlass* vk, GrowableArray<ciType*>& visited) {
1300 // Create a new InlineTypeNode initialized with all zero
1301 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT), /* null_free= */ true);
1302 vt->set_is_buffered(gvn, false);
1303 vt->set_null_marker(gvn);
1304 for (uint i = 0; i < vt->field_count(); ++i) {
1305 ciField* field = vt->field(i);
1306 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1307 ciType* ft = field->type();
1308 Node* value = gvn.zerocon(ft->basic_type());
1309 if (!field->is_flat() && visited.contains(ft)) {
1310 gvn.C->set_has_circular_inline_type(true);
1311 } else if (ft->is_inlinetype()) {
1312 int old_len = visited.length();
1313 visited.push(ft);
1314 ciInlineKlass* vk_field = ft->as_inline_klass();
1315 if (field->is_null_free()) {
1316 value = make_all_zero_impl(gvn, vk_field, visited);
1317 } else {
1318 value = make_null_impl(gvn, vk_field, visited);
1319 }
1320 visited.trunc_to(old_len);
1321 }
1322 vt->set_field_value(i, value);
1323 }
1324 vt = gvn.transform(vt)->as_InlineType();
1325 assert(vt->is_all_zero(&gvn), "must be the all-zero inline type");
1326 return vt;
1327 }
1328
1329 bool InlineTypeNode::is_all_zero(PhaseGVN* gvn, bool flat) const {
1330 const TypeInt* tinit = gvn->type(get_null_marker())->isa_int();
1331 if (tinit == nullptr || !tinit->is_con(1)) {
1332 return false; // May be null
1333 }
1334 for (uint i = 0; i < field_count(); ++i) {
1335 Node* value = field_value(i);
1336 ciField* field = this->field(i);
1337 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1338 if (field->is_null_free()) {
1339 // Null-free value class field must have the all-zero value. If 'flat' is set,
1340 // reject non-flat fields because they need to be initialized with an oop to a buffer.
1341 if (!value->is_InlineType() || !value->as_InlineType()->is_all_zero(gvn) || (flat && !field->is_flat())) {
1342 return false;
1343 }
1344 continue;
1345 } else if (value->is_InlineType()) {
1346 // Nullable value class field must be null
1347 tinit = gvn->type(value->as_InlineType()->get_null_marker())->isa_int();
1348 if (tinit != nullptr && tinit->is_con(0)) {
1349 continue;
1350 }
1351 return false;
1352 } else if (!gvn->type(value)->is_zero_type()) {
1353 return false;
1354 }
1355 }
1356 return true;
1357 }
1358
1359 InlineTypeNode* InlineTypeNode::make_from_oop(GraphKit* kit, Node* oop, ciInlineKlass* vk) {
1360 GrowableArray<ciType*> visited;
1361 visited.push(vk);
1362 return make_from_oop_impl(kit, oop, vk, visited);
1363 }
1364
1365 InlineTypeNode* InlineTypeNode::make_from_oop_impl(GraphKit* kit, Node* oop, ciInlineKlass* vk, GrowableArray<ciType*>& visited) {
1366 PhaseGVN& gvn = kit->gvn();
1367
1368 // Create and initialize an InlineTypeNode by loading all field
1369 // values from a heap-allocated version and also save the oop.
1370 InlineTypeNode* vt = oop->isa_InlineType();
1371 if (vt != nullptr) {
1372 return vt;
1373 }
1374
1375 if (gvn.type(oop)->maybe_null()) {
1376 // Add a null check because the oop may be null
1377 Node* null_ctl = kit->top();
1378 Node* not_null_oop = kit->null_check_oop(oop, &null_ctl);
1379 if (kit->stopped()) {
1380 // Constant null
1381 kit->set_control(null_ctl);
1382 vt = make_null_impl(gvn, vk, visited);
1383 kit->record_for_igvn(vt);
1384 return vt;
1385 }
1386 vt = new InlineTypeNode(vk, not_null_oop, /* null_free= */ false);
1387 vt->set_is_buffered(gvn);
1388 vt->set_null_marker(gvn);
1389 Node* payload_ptr = kit->basic_plus_adr(not_null_oop, vk->payload_offset());
1390 vt->load(kit, not_null_oop, payload_ptr, true, true, IN_HEAP | MO_UNORDERED, visited);
1391
1392 if (null_ctl != kit->top()) {
1393 InlineTypeNode* null_vt = make_null_impl(gvn, vk, visited);
1394 Node* region = new RegionNode(3);
1395 region->init_req(1, kit->control());
1396 region->init_req(2, null_ctl);
1397 vt = vt->clone_with_phis(&gvn, region, kit->map());
1398 vt->merge_with(&gvn, null_vt, 2, true);
1399 vt->set_oop(gvn, oop);
1400 vt->set_is_buffered(gvn);
1401 kit->set_control(gvn.transform(region));
1402 }
1403 } else {
1404 // Oop can never be null
1405 vt = new InlineTypeNode(vk, oop, /* null_free= */ true);
1406 vt->set_is_buffered(gvn);
1407 vt->set_null_marker(gvn);
1408 Node* payload_ptr = kit->basic_plus_adr(oop, vk->payload_offset());
1409 vt->load(kit, oop, payload_ptr, true, true, IN_HEAP | MO_UNORDERED, visited);
1410 }
1411 assert(vt->is_allocated(&gvn), "inline type should be allocated");
1412 kit->record_for_igvn(vt);
1413 return gvn.transform(vt)->as_InlineType();
1414 }
1415
1416 InlineTypeNode* InlineTypeNode::make_from_flat(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr,
1417 bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators) {
1418 GrowableArray<ciType*> visited;
1419 visited.push(vk);
1420 return make_from_flat_impl(kit, vk, base, ptr, atomic, immutable_memory, null_free, null_free, decorators, visited);
1421 }
1422
1423 // GraphKit wrapper for the 'make_from_flat' method
1424 InlineTypeNode* InlineTypeNode::make_from_flat_impl(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr, bool atomic, bool immutable_memory,
1425 bool null_free, bool trust_null_free_oop, DecoratorSet decorators, GrowableArray<ciType*>& visited) {
1426 assert(null_free || !trust_null_free_oop, "cannot trust null-free oop when the holder object is not null-free");
1427 PhaseGVN& gvn = kit->gvn();
1428 bool do_atomic = atomic;
1429 // With immutable memory, a non-atomic load and an atomic load are the same
1430 if (immutable_memory) {
1431 do_atomic = false;
1432 }
1433 // If there is only one flattened field, a non-atomic load and an atomic load are the same
1434 if (vk->is_naturally_atomic(null_free)) {
1435 do_atomic = false;
1436 }
1437
1438 if (!do_atomic) {
1439 InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk, null_free);
1440 if (!null_free) {
1441 int nm_offset = vk->null_marker_offset_in_payload();
1442 Node* nm_ptr = kit->basic_plus_adr(base, ptr, nm_offset);
1443 const TypePtr* nm_ptr_type = (decorators & C2_MISMATCHED) == 0 ? gvn.type(nm_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
1444 Node* nm_value = kit->access_load_at(base, nm_ptr, nm_ptr_type, TypeInt::BOOL, T_BOOLEAN, decorators);
1445 vt->set_req(NullMarker, nm_value);
1446 }
1447
1448 vt->load(kit, base, ptr, immutable_memory, trust_null_free_oop, decorators, visited);
1449 return gvn.transform(vt)->as_InlineType();
1450 }
1451
1452 assert(!immutable_memory, "immutable memory does not need explicit atomic access");
1453 return LoadFlatNode::load(kit, vk, base, ptr, null_free, trust_null_free_oop, decorators);
1454 }
1455
1456 InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* idx) {
1457 assert(vk->maybe_flat_in_array(), "element type %s cannot be flat in array", vk->name()->as_utf8());
1458 PhaseGVN& gvn = kit->gvn();
1459 // The flat field loads are dependent on both the array layout checks as well as the range check.
1460 DecoratorSet decorators = IN_HEAP | IS_ARRAY | MO_UNORDERED | C2_CONTROL_DEPENDENT_LOAD | C2_UNKNOWN_CONTROL_LOAD;
1461 kit->C->set_flat_accesses();
1462 InlineTypeNode* vt_nullable = nullptr;
1463 InlineTypeNode* vt_null_free = nullptr;
1464 InlineTypeNode* vt_non_atomic = nullptr;
1465
1466 RegionNode* region = new RegionNode(4);
1467 gvn.set_type(region, Type::CONTROL);
1468 kit->record_for_igvn(region);
1469
1470 Node* input_memory_state = kit->reset_memory();
1471 kit->set_all_memory(input_memory_state);
1472
1473 PhiNode* mem = PhiNode::make(region, input_memory_state, Type::MEMORY, TypePtr::BOTTOM);
1474 gvn.set_type(mem, Type::MEMORY);
1475 kit->record_for_igvn(mem);
1476
1477 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO);
1478 gvn.set_type(io, Type::ABIO);
1479 kit->record_for_igvn(io);
1480
1481 Node* bol_null_free = kit->null_free_array_test(base); // Argument evaluation order is undefined in C++ and since this sets control, it needs to come first
1482 IfNode* iff_null_free = kit->create_and_map_if(kit->control(), bol_null_free, PROB_FAIR, COUNT_UNKNOWN);
1483
1484 // Nullable
1485 kit->set_control(kit->IfFalse(iff_null_free));
1486 if (!kit->stopped()) {
1487 assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8());
1488 kit->set_all_memory(input_memory_state);
1489 Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true);
1490 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
1491 vt_nullable = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, false, decorators);
1492
1493 region->init_req(1, kit->control());
1494 mem->set_req(1, kit->reset_memory());
1495 io->set_req(1, kit->i_o());
1496 }
1497
1498 // Null-free
1499 kit->set_control(kit->IfTrue(iff_null_free));
1500 if (!kit->stopped()) {
1501 kit->set_all_memory(input_memory_state);
1502
1503 Node* bol_atomic = kit->null_free_atomic_array_test(base, vk);
1504 IfNode* iff_atomic = kit->create_and_map_if(kit->control(), bol_atomic, PROB_FAIR, COUNT_UNKNOWN);
1505
1506 // Atomic
1507 kit->set_control(kit->IfTrue(iff_atomic));
1508 if (!kit->stopped()) {
1509 assert(vk->has_null_free_atomic_layout(), "element type %s does not have a null-free atomic flat layout", vk->name()->as_utf8());
1510 kit->set_all_memory(input_memory_state);
1511 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, true);
1512 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
1513 vt_null_free = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, true, decorators);
1514
1515 region->init_req(2, kit->control());
1516 mem->set_req(2, kit->reset_memory());
1517 io->set_req(2, kit->i_o());
1518 }
1519
1520 // Non-Atomic
1521 kit->set_control(kit->IfFalse(iff_atomic));
1522 if (!kit->stopped()) {
1523 assert(vk->has_null_free_non_atomic_layout(), "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8());
1524 kit->set_all_memory(input_memory_state);
1525 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, false);
1526 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
1527 vt_non_atomic = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, false, false, true, decorators);
1528
1529 region->init_req(3, kit->control());
1530 mem->set_req(3, kit->reset_memory());
1531 io->set_req(3, kit->i_o());
1532 }
1533 }
1534
1535 InlineTypeNode* vt = nullptr;
1536 if (vt_nullable == nullptr && vt_null_free == nullptr && vt_non_atomic == nullptr) {
1537 // All paths are dead
1538 vt = make_null(gvn, vk);
1539 } else if (vt_nullable == nullptr && vt_null_free == nullptr) {
1540 vt = vt_non_atomic;
1541 } else if (vt_nullable == nullptr && vt_non_atomic == nullptr) {
1542 vt = vt_null_free;
1543 } else if (vt_null_free == nullptr && vt_non_atomic == nullptr) {
1544 vt = vt_nullable;
1545 }
1546 if (vt != nullptr) {
1547 kit->set_control(kit->gvn().transform(region));
1548 kit->set_all_memory(kit->gvn().transform(mem));
1549 kit->set_i_o(kit->gvn().transform(io));
1550 return vt;
1551 }
1552
1553 InlineTypeNode* zero = InlineTypeNode::make_null(gvn, vk);
1554 vt = zero->clone_with_phis(&gvn, region);
1555 if (vt_nullable != nullptr) {
1556 vt = vt->merge_with(&gvn, vt_nullable, 1, false);
1557 }
1558 if (vt_null_free != nullptr) {
1559 vt = vt->merge_with(&gvn, vt_null_free, 2, false);
1560 }
1561 if (vt_non_atomic != nullptr) {
1562 vt = vt->merge_with(&gvn, vt_non_atomic, 3, false);
1563 }
1564
1565 kit->set_control(kit->gvn().transform(region));
1566 kit->set_all_memory(kit->gvn().transform(mem));
1567 kit->set_i_o(kit->gvn().transform(io));
1568 return gvn.transform(vt)->as_InlineType();
1569 }
1570
1571 InlineTypeNode* InlineTypeNode::make_from_multi(GraphKit* kit, MultiNode* multi, ciInlineKlass* vk, uint& base_input, bool in, bool null_free) {
1572 InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk, null_free);
1573 if (!in || multi->is_Start()) {
1574 // Keep track of the oop. The inline type might already be buffered.
1575 Node* oop = nullptr;
1576 if (multi->is_Start()) {
1577 oop = kit->gvn().transform(new ParmNode(multi->as_Start(), base_input++));
1578 } else {
1579 oop = kit->gvn().transform(new ProjNode(multi, base_input++));
1580 }
1581 vt->set_oop(kit->gvn(), oop);
1582 } else {
1583 Node* oop = multi->as_Call()->in(base_input++);
1584 vt->set_oop(kit->gvn(), oop);
1585 }
1586 GrowableArray<ciType*> visited;
1587 visited.push(vk);
1588 vt->initialize_fields(kit, multi, base_input, in, null_free, nullptr, visited);
1589 return kit->gvn().transform(vt)->as_InlineType();
1590 }
1591
1592 Node* InlineTypeNode::is_loaded(PhaseGVN* phase, ciInlineKlass* vk, Node* base, int holder_offset) const {
1593 if (vk == nullptr) {
1594 vk = inline_klass();
1595 }
1596 for (uint i = 0; i < field_count(); ++i) {
1597 ciField* field = this->field(i);
1598 int offset = holder_offset + field->offset_in_bytes();
1599 Node* value = field_value(i);
1600 if (value->is_InlineType()) {
1601 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1602 InlineTypeNode* vt = value->as_InlineType();
1603 if (vt->type()->inline_klass()->is_empty()) {
1604 continue;
1605 } else if (field->is_flat() && vt->is_InlineType()) {
1606 // Check inline type field load recursively
1607 base = vt->as_InlineType()->is_loaded(phase, vk, base, offset - vt->type()->inline_klass()->payload_offset());
1608 if (base == nullptr) {
1609 return nullptr;
1610 }
1611 continue;
1612 } else {
1613 value = vt->get_oop();
1614 if (value->Opcode() == Op_CastPP) {
1615 // Skip CastPP
1616 value = value->in(1);
1617 }
1618 }
1619 }
1620 if (value->isa_DecodeN()) {
1621 // Skip DecodeN
1622 value = value->in(1);
1623 }
1624 if (value->isa_Load()) {
1625 // Check if base and offset of field load matches inline type layout
1626 intptr_t loffset = 0;
1627 Node* lbase = AddPNode::Ideal_base_and_offset(value->in(MemNode::Address), phase, loffset);
1628 if (lbase == nullptr || (lbase != base && base != nullptr) || loffset != offset) {
1629 return nullptr;
1630 } else if (base == nullptr) {
1631 // Set base and check if pointer type matches
1632 base = lbase;
1633 const TypeInstPtr* vtptr = phase->type(base)->isa_instptr();
1634 if (vtptr == nullptr || !vtptr->instance_klass()->equals(vk)) {
1635 return nullptr;
1636 }
1637 }
1638 } else {
1639 return nullptr;
1640 }
1641 }
1642 return base;
1643 }
1644
1645 Node* InlineTypeNode::tagged_klass(ciInlineKlass* vk, PhaseGVN& gvn) {
1646 const TypeKlassPtr* tk = TypeKlassPtr::make(vk);
1647 intptr_t bits = tk->get_con();
1648 set_nth_bit(bits, 0);
1649 return gvn.longcon((jlong)bits);
1650 }
1651
1652 void InlineTypeNode::pass_fields(GraphKit* kit, Node* n, uint& base_input, bool in, bool null_free, bool root) {
1653 if (root) {
1654 if (is_allocated(&kit->gvn())) {
1655 // Keep the information that 'this' is buffered
1656 n->init_req(base_input++, this);
1657 } else {
1658 n->init_req(base_input++, get_oop());
1659 }
1660 }
1661 if (!null_free && in) {
1662 n->init_req(base_input++, get_null_marker());
1663 }
1664 for (uint i = 0; i < field_count(); i++) {
1665 Node* arg = field_value(i);
1666 ciField* field = this->field(i);
1667 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1668 if (field->is_flat()) {
1669 // Flat inline type field
1670 arg->as_InlineType()->pass_fields(kit, n, base_input, in);
1671 if (!field->is_null_free()) {
1672 assert(field->null_marker_offset() != -1, "inconsistency");
1673 n->init_req(base_input++, arg->as_InlineType()->get_null_marker());
1674 }
1675 } else {
1676 if (arg->is_InlineType()) {
1677 // Non-flat inline type field
1678 InlineTypeNode* vt = arg->as_InlineType();
1679 assert(n->Opcode() != Op_Return || vt->is_allocated(&kit->gvn()), "inline type field should be allocated on return");
1680 arg = vt->buffer(kit);
1681 }
1682 // Initialize call/return arguments
1683 n->init_req(base_input++, arg);
1684 if (field->type()->size() == 2) {
1685 n->init_req(base_input++, kit->top());
1686 }
1687 }
1688 }
1689 }
1690
1691 void InlineTypeNode::initialize_fields(GraphKit* kit, MultiNode* multi, uint& base_input, bool in, bool no_null_marker, Node* null_check_region, GrowableArray<ciType*>& visited) {
1692 PhaseGVN& gvn = kit->gvn();
1693 Node* null_marker = nullptr;
1694 if (!no_null_marker) {
1695 // Nullable inline type
1696 if (in) {
1697 // Set null marker
1698 if (multi->is_Start()) {
1699 null_marker = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1700 } else {
1701 null_marker = multi->as_Call()->in(base_input);
1702 }
1703 set_req(NullMarker, null_marker);
1704 base_input++;
1705 }
1706 // Add a null check to make subsequent loads dependent on
1707 assert(null_check_region == nullptr, "already set");
1708 if (null_marker == nullptr) {
1709 // Will only be initialized below, use dummy node for now
1710 null_marker = new Node(1);
1711 null_marker->init_req(0, kit->control()); // Add an input to prevent dummy from being dead
1712 gvn.set_type_bottom(null_marker);
1713 }
1714 Node* null_ctrl = kit->top();
1715 kit->null_check_common(null_marker, T_INT, false, &null_ctrl);
1716 Node* non_null_ctrl = kit->control();
1717 null_check_region = new RegionNode(3);
1718 null_check_region->init_req(1, non_null_ctrl);
1719 null_check_region->init_req(2, null_ctrl);
1720 null_check_region = gvn.transform(null_check_region);
1721 kit->set_control(null_check_region);
1722 }
1723
1724 for (uint i = 0; i < field_count(); ++i) {
1725 ciField* field = this->field(i);
1726 ciType* type = field->type();
1727 Node* parm = nullptr;
1728 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1729 if (field->is_flat()) {
1730 // Flat inline type field
1731 InlineTypeNode* vt = make_uninitialized(gvn, type->as_inline_klass(), field->is_null_free());
1732 vt->initialize_fields(kit, multi, base_input, in, true, null_check_region, visited);
1733 if (!field->is_null_free()) {
1734 assert(field->null_marker_offset() != -1, "inconsistency");
1735 Node* null_marker_field_vt = nullptr;
1736 if (multi->is_Start()) {
1737 null_marker_field_vt = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1738 } else if (in) {
1739 null_marker_field_vt = multi->as_Call()->in(base_input);
1740 } else {
1741 null_marker_field_vt = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1742 }
1743 vt->set_req(NullMarker, null_marker_field_vt);
1744 base_input++;
1745 }
1746 parm = gvn.transform(vt);
1747 } else {
1748 if (multi->is_Start()) {
1749 assert(in, "return from start?");
1750 parm = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1751 } else if (in) {
1752 parm = multi->as_Call()->in(base_input);
1753 } else {
1754 parm = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1755 }
1756 // Non-flat inline type field
1757 if (type->is_inlinetype()) {
1758 if (null_check_region != nullptr) {
1759 // We limit scalarization for inline types with circular fields and can therefore observe nodes
1760 // of the same type but with different scalarization depth during GVN. To avoid inconsistencies
1761 // during merging, make sure that we only create Phis for fields that are guaranteed to be scalarized.
1762 if (parm->is_InlineType() && kit->C->has_circular_inline_type()) {
1763 parm = parm->as_InlineType()->get_oop();
1764 }
1765 // Holder is nullable, set field to nullptr if holder is nullptr to avoid loading from uninitialized memory
1766 parm = PhiNode::make(null_check_region, parm, TypeInstPtr::make(TypePtr::BotPTR, type->as_inline_klass()));
1767 parm->set_req(2, kit->zerocon(T_OBJECT));
1768 parm = gvn.transform(parm);
1769 }
1770 if (visited.contains(type)) {
1771 kit->C->set_has_circular_inline_type(true);
1772 } else if (!parm->is_InlineType()) {
1773 int old_len = visited.length();
1774 visited.push(type);
1775 parm = make_from_oop_impl(kit, parm, type->as_inline_klass(), visited);
1776 visited.trunc_to(old_len);
1777 }
1778 }
1779 base_input += type->size();
1780 }
1781 assert(parm != nullptr, "should never be null");
1782 assert(field_value(i) == nullptr, "already set");
1783 set_field_value(i, parm);
1784 gvn.record_for_igvn(parm);
1785 }
1786 // The last argument is used to pass the null marker to compiled code
1787 if (!no_null_marker && !in) {
1788 Node* cmp = null_marker->raw_out(0);
1789 null_marker = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1790 set_req(NullMarker, null_marker);
1791 gvn.hash_delete(cmp);
1792 cmp->set_req(1, null_marker);
1793 gvn.hash_find_insert(cmp);
1794 gvn.record_for_igvn(cmp);
1795 base_input++;
1796 }
1797 }
1798
1799 // Search for multiple allocations of this inline type and try to replace them by dominating allocations.
1800 // Equivalent InlineTypeNodes are merged by GVN, so we just need to search for AllocateNode users to find redundant allocations.
1801 void InlineTypeNode::remove_redundant_allocations(PhaseIdealLoop* phase) const {
1802 PhaseIterGVN* igvn = &phase->igvn();
1803 // Search for allocations of this inline type. Ignore scalar replaceable ones, they
1804 // will be removed anyway and changing the memory chain will confuse other optimizations.
1805 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1806 AllocateNode* alloc = fast_out(i)->isa_Allocate();
1807 if (alloc != nullptr && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable) {
1808 Node* res = alloc->result_cast();
1809 if (res == nullptr || !res->is_CheckCastPP()) {
1810 break; // No unique CheckCastPP
1811 }
1812 // Search for a dominating allocation of the same inline type
1813 Node* res_dom = res;
1814 for (DUIterator_Fast jmax, j = fast_outs(jmax); j < jmax; j++) {
1815 AllocateNode* alloc_other = fast_out(j)->isa_Allocate();
1816 if (alloc_other != nullptr && alloc_other->in(AllocateNode::InlineType) == this && !alloc_other->_is_scalar_replaceable) {
1817 Node* res_other = alloc_other->result_cast();
1818 if (res_other != nullptr && res_other->is_CheckCastPP() && res_other != res_dom &&
1819 phase->is_dominator(res_other->in(0), res_dom->in(0))) {
1820 res_dom = res_other;
1821 }
1822 }
1823 }
1824 if (res_dom != res) {
1825 // Replace allocation by dominating one.
1826 replace_allocation(igvn, res, res_dom);
1827 // The result of the dominated allocation is now unused and will be removed
1828 // later in PhaseMacroExpand::eliminate_allocate_node to not confuse loop opts.
1829 igvn->_worklist.push(alloc);
1830 }
1831 }
1832 }
1833 }
1834
1835 InlineTypeNode* InlineTypeNode::make_null(PhaseGVN& gvn, ciInlineKlass* vk, bool transform) {
1836 GrowableArray<ciType*> visited;
1837 visited.push(vk);
1838 return make_null_impl(gvn, vk, visited, transform);
1839 }
1840
1841 InlineTypeNode* InlineTypeNode::make_null_impl(PhaseGVN& gvn, ciInlineKlass* vk, GrowableArray<ciType*>& visited, bool transform) {
1842 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT), /* null_free= */ false);
1843 vt->set_is_buffered(gvn);
1844 vt->set_null_marker(gvn, gvn.intcon(0));
1845 for (uint i = 0; i < vt->field_count(); i++) {
1846 ciField* field = vt->field(i);
1847 ciType* ft = field->type();
1848 Node* value = gvn.zerocon(ft->basic_type());
1849 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1850 if (!field->is_flat() && visited.contains(ft)) {
1851 gvn.C->set_has_circular_inline_type(true);
1852 } else if (ft->is_inlinetype()) {
1853 int old_len = visited.length();
1854 visited.push(ft);
1855 value = make_null_impl(gvn, ft->as_inline_klass(), visited);
1856 visited.trunc_to(old_len);
1857 }
1858 vt->set_field_value(i, value);
1859 }
1860 return transform ? gvn.transform(vt)->as_InlineType() : vt;
1861 }
1862
1863 InlineTypeNode* InlineTypeNode::clone_if_required(PhaseGVN* gvn, SafePointNode* map, bool safe_for_replace) {
1864 if (!safe_for_replace || (map == nullptr && outcnt() != 0)) {
1865 return clone()->as_InlineType();
1866 }
1867 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1868 if (fast_out(i) != map) {
1869 return clone()->as_InlineType();
1870 }
1871 }
1872 gvn->hash_delete(this);
1873 return this;
1874 }
1875
1876 const Type* InlineTypeNode::Value(PhaseGVN* phase) const {
1877 Node* oop = get_oop();
1878 const Type* toop = phase->type(oop);
1879 #ifdef ASSERT
1880 if (oop->is_Con() && toop->is_zero_type() && _type->isa_oopptr()->is_known_instance()) {
1881 // We are not allocated (anymore) and should therefore not have an instance id
1882 dump(1);
1883 assert(false, "Unbuffered inline type should not have known instance id");
1884 }
1885 #endif
1886 if (toop == Type::TOP) {
1887 return Type::TOP;
1888 }
1889 const Type* t = toop->filter_speculative(_type);
1890 // Because of contradicting type profiling, we can end up with top as speculative type,
1891 // which would then get removed by cleanup_speculative. In this case we have to run filter_speculative
1892 // again, otherwise we would break the idempotence of Value
1893 if (t->speculative() == nullptr && toop->speculative() != nullptr) {
1894 t = toop->filter_speculative(t);
1895 }
1896 if (t->singleton()) {
1897 // Don't replace InlineType by a constant
1898 t = _type;
1899 }
1900 const Type* tinit = phase->type(in(NullMarker));
1901 if (tinit == Type::TOP) {
1902 return Type::TOP;
1903 }
1904 if (tinit->isa_int() && tinit->is_int()->is_con(1)) {
1905 t = t->join_speculative(TypePtr::NOTNULL);
1906 }
1907 return t;
1908 }
1909
1910 InlineTypeNode* LoadFlatNode::load(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr, bool null_free, bool trust_null_free_oop, DecoratorSet decorators) {
1911 int output_type_size = vk->nof_nonstatic_fields() + (null_free ? 0 : 1);
1912 const Type** output_types = TypeTuple::fields(output_type_size);
1913 collect_field_types(vk, output_types + TypeFunc::Parms, 0, output_type_size, null_free, trust_null_free_oop);
1914 const TypeTuple* type = TypeTuple::make(output_type_size + TypeFunc::Parms, output_types);
1915
1916 LoadFlatNode* load = new LoadFlatNode(vk, type, null_free, decorators);
1917 load->init_req(TypeFunc::Control, kit->control());
1918 load->init_req(TypeFunc::I_O, kit->top());
1919 load->init_req(TypeFunc::Memory, kit->reset_memory());
1920 load->init_req(TypeFunc::FramePtr, kit->frameptr());
1921 load->init_req(TypeFunc::ReturnAdr, kit->top());
1922
1923 load->init_req(TypeFunc::Parms, base);
1924 load->init_req(TypeFunc::Parms + 1, ptr);
1925 kit->kill_dead_locals();
1926 kit->add_safepoint_edges(load);
1927 load = kit->gvn().transform(load)->as_LoadFlat();
1928 kit->record_for_igvn(load);
1929
1930 kit->set_control(kit->gvn().transform(new ProjNode(load, TypeFunc::Control)));
1931 kit->set_all_memory(kit->gvn().transform(new ProjNode(load, TypeFunc::Memory)));
1932 return load->collect_projs(kit, vk, TypeFunc::Parms, null_free);
1933 }
1934
1935 bool LoadFlatNode::expand_constant(PhaseIterGVN& igvn, ciInstance* inst) const {
1936 precond(inst != nullptr);
1937 assert(igvn.delay_transform(), "transformation must be delayed");
1938 if ((_decorators & C2_MISMATCHED) != 0) {
1939 return false;
1940 }
1941
1942 GraphKit kit(this, igvn);
1943 for (int i = 0; i < _vk->nof_nonstatic_fields(); i++) {
1944 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + i);
1945 if (proj_out == nullptr) {
1946 continue;
1947 }
1948
1949 ciField* field = _vk->nonstatic_field_at(i);
1950 BasicType bt = field->type()->basic_type();
1951 if (inst == nullptr) {
1952 Node* cst_node = igvn.zerocon(bt);
1953 igvn.replace_node(proj_out, cst_node);
1954 } else {
1955 bool is_unsigned_load = bt == T_BOOLEAN || bt == T_CHAR;
1956 const Type* cst_type = Type::make_constant_from_field(field, inst, bt, is_unsigned_load);
1957 Node* cst_node = igvn.makecon(cst_type);
1958 igvn.replace_node(proj_out, cst_node);
1959 }
1960 }
1961
1962 if (!_null_free) {
1963 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + _vk->nof_nonstatic_fields());
1964 if (proj_out != nullptr) {
1965 igvn.replace_node(proj_out, igvn.intcon(1));
1966 }
1967 }
1968
1969 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
1970 if (old_ctrl != nullptr) {
1971 igvn.replace_node(old_ctrl, kit.control());
1972 }
1973 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
1974 Node* new_mem = kit.reset_memory();
1975 if (old_mem != nullptr) {
1976 igvn.replace_node(old_mem, new_mem);
1977 }
1978 return true;
1979 }
1980
1981 bool LoadFlatNode::expand_non_atomic(PhaseIterGVN& igvn) const {
1982 assert(igvn.delay_transform(), "transformation must be delayed");
1983 if ((_decorators & C2_MISMATCHED) != 0) {
1984 return false;
1985 }
1986
1987 GraphKit kit(this, igvn);
1988 Node* base = this->base();
1989 Node* ptr = this->ptr();
1990
1991 for (int i = 0; i < _vk->nof_nonstatic_fields(); i++) {
1992 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + i);
1993 if (proj_out == nullptr) {
1994 continue;
1995 }
1996
1997 ciField* field = _vk->nonstatic_field_at(i);
1998 Node* field_ptr = kit.basic_plus_adr(base, ptr, field->offset_in_bytes() - _vk->payload_offset());
1999 const TypePtr* field_ptr_type = field_ptr->Value(&igvn)->is_ptr();
2000 igvn.set_type(field_ptr, field_ptr_type);
2001
2002 Node* field_value = kit.access_load_at(base, field_ptr, field_ptr_type, igvn.type(proj_out), field->type()->basic_type(), _decorators);
2003 igvn.replace_node(proj_out, field_value);
2004 }
2005
2006 if (!_null_free) {
2007 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + _vk->nof_nonstatic_fields());
2008 if (proj_out != nullptr) {
2009 Node* null_marker_ptr = kit.basic_plus_adr(base, ptr, _vk->null_marker_offset_in_payload());
2010 const TypePtr* null_marker_ptr_type = null_marker_ptr->Value(&igvn)->is_ptr();
2011 igvn.set_type(null_marker_ptr, null_marker_ptr_type);
2012 Node* null_marker_value = kit.access_load_at(base, null_marker_ptr, null_marker_ptr_type, TypeInt::BOOL, T_BOOLEAN, _decorators);
2013 igvn.replace_node(proj_out, null_marker_value);
2014 }
2015 }
2016
2017 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
2018 if (old_ctrl != nullptr) {
2019 igvn.replace_node(old_ctrl, kit.control());
2020 }
2021 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
2022 Node* new_mem = kit.reset_memory();
2023 if (old_mem != nullptr) {
2024 igvn.replace_node(old_mem, new_mem);
2025 }
2026 return true;
2027 }
2028
2029 void LoadFlatNode::expand_atomic(PhaseIterGVN& igvn) const {
2030 assert(igvn.delay_transform(), "transformation must be delayed");
2031 GraphKit kit(this, igvn);
2032 Node* base = this->base();
2033 Node* ptr = this->ptr();
2034
2035 BasicType payload_bt = _vk->atomic_size_to_basic_type(_null_free);
2036 kit.insert_mem_bar(Op_MemBarCPUOrder);
2037 Node* payload = kit.access_load_at(base, ptr, TypeRawPtr::BOTTOM, Type::get_const_basic_type(payload_bt), payload_bt,
2038 _decorators | C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD | C2_UNKNOWN_CONTROL_LOAD, kit.control());
2039 kit.insert_mem_bar(Op_MemBarCPUOrder);
2040
2041 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
2042 if (old_ctrl != nullptr) {
2043 igvn.replace_node(old_ctrl, kit.control());
2044 }
2045 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
2046 Node* new_mem = kit.reset_memory();
2047 if (old_mem != nullptr) {
2048 igvn.replace_node(old_mem, new_mem);
2049 }
2050
2051 expand_projs_atomic(igvn, kit.control(), payload);
2052 }
2053
2054 void LoadFlatNode::collect_field_types(ciInlineKlass* vk, const Type** field_types, int idx, int limit, bool null_free, bool trust_null_free_oop) {
2055 assert(null_free || !trust_null_free_oop, "cannot trust null-free oop when the holder object is not null-free");
2056 for (int i = 0; i < vk->nof_declared_nonstatic_fields(); i++) {
2057 ciField* field = vk->declared_nonstatic_field_at(i);
2058 if (field->is_flat()) {
2059 ciInlineKlass* field_klass = field->type()->as_inline_klass();
2060 collect_field_types(field_klass, field_types, idx, limit, field->is_null_free(), trust_null_free_oop && field->is_null_free());
2061 idx += field_klass->nof_nonstatic_fields() + (field->is_null_free() ? 0 : 1);
2062 continue;
2063 }
2064
2065 const Type* field_type = Type::get_const_type(field->type());
2066 if (trust_null_free_oop && field->is_null_free()) {
2067 field_type = field_type->filter(TypePtr::NOTNULL);
2068 }
2069
2070 assert(idx >= 0 && idx < limit, "field type out of bounds, %d - %d", idx, limit);
2071 field_types[idx] = field_type;
2072 idx++;
2073 }
2074
2075 if (!null_free) {
2076 assert(idx >= 0 && idx < limit, "field type out of bounds, %d - %d", idx, limit);
2077 field_types[idx] = TypeInt::BOOL;
2078 }
2079 }
2080
2081 // Create an InlineTypeNode from a LoadFlatNode with its fields being extracted from the
2082 // LoadFlatNode
2083 InlineTypeNode* LoadFlatNode::collect_projs(GraphKit* kit, ciInlineKlass* vk, int proj_con, bool null_free) {
2084 PhaseGVN& gvn = kit->gvn();
2085 InlineTypeNode* res = InlineTypeNode::make_uninitialized(gvn, vk, null_free);
2086 for (int i = 0; i < vk->nof_declared_nonstatic_fields(); i++) {
2087 ciField* field = vk->declared_nonstatic_field_at(i);
2088 Node* field_value;
2089 if (field->is_flat()) {
2090 ciInlineKlass* field_klass = field->type()->as_inline_klass();
2091 field_value = collect_projs(kit, field_klass, proj_con, field->is_null_free());
2092 proj_con += field_klass->nof_nonstatic_fields() + (field->is_null_free() ? 0 : 1);
2093 } else {
2094 field_value = gvn.transform(new ProjNode(this, proj_con));
2095 if (field->type()->is_inlinetype()) {
2096 field_value = InlineTypeNode::make_from_oop(kit, field_value, field->type()->as_inline_klass());
2097 }
2098 proj_con++;
2099 }
2100 res->set_field_value(i, field_value);
2101 }
2102
2103 if (null_free) {
2104 res->set_null_marker(gvn);
2105 } else {
2106 res->set_null_marker(gvn, gvn.transform(new ProjNode(this, proj_con)));
2107 }
2108 return gvn.transform(res)->as_InlineType();
2109 }
2110
2111 // Extract the values of the flattened fields from the loaded payload
2112 void LoadFlatNode::expand_projs_atomic(PhaseIterGVN& igvn, Node* ctrl, Node* payload) const {
2113 BasicType payload_bt = _vk->atomic_size_to_basic_type(_null_free);
2114 for (int i = 0; i < _vk->nof_nonstatic_fields(); i++) {
2115 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + i);
2116 if (proj_out == nullptr) {
2117 continue;
2118 }
2119
2120 ciField* field = _vk->nonstatic_field_at(i);
2121 int field_offset = field->offset_in_bytes() - _vk->payload_offset();
2122 const Type* field_type = igvn.type(proj_out);
2123 Node* field_value = get_payload_value(igvn, ctrl, payload_bt, payload, field_type, field->type()->basic_type(), field_offset);
2124 igvn.replace_node(proj_out, field_value);
2125 }
2126
2127 if (!_null_free) {
2128 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + _vk->nof_nonstatic_fields());
2129 if (proj_out == nullptr) {
2130 return;
2131 }
2132
2133 int null_marker_offset = _vk->null_marker_offset_in_payload();
2134 Node* null_marker_value = get_payload_value(igvn, ctrl, payload_bt, payload, TypeInt::BOOL, T_BOOLEAN, null_marker_offset);
2135 igvn.replace_node(proj_out, null_marker_value);
2136 }
2137 }
2138
2139 Node* LoadFlatNode::get_payload_value(PhaseIterGVN& igvn, Node* ctrl, BasicType payload_bt, Node* payload, const Type* value_type, BasicType value_bt, int offset) {
2140 assert((offset + type2aelembytes(value_bt)) <= type2aelembytes(payload_bt), "Value does not fit into payload");
2141 Node* value = nullptr;
2142 // Shift to the right position in the long value
2143 Node* shift_val = igvn.intcon(offset << LogBitsPerByte);
2144 if (payload_bt == T_LONG) {
2145 value = igvn.transform(new URShiftLNode(payload, shift_val));
2146 value = igvn.transform(new ConvL2INode(value));
2147 } else {
2148 value = igvn.transform(new URShiftINode(payload, shift_val));
2149 }
2150
2151 if (value_bt == T_INT) {
2152 return value;
2153 } else if (!is_java_primitive(value_bt)) {
2154 assert(UseCompressedOops && payload_bt == T_LONG, "Naturally atomic");
2155 value = igvn.transform(new CastI2NNode(ctrl, value, value_type->make_narrowoop()));
2156 value = igvn.transform(new DecodeNNode(value, value_type));
2157
2158 // Similar to CheckCastPP nodes with raw input, CastI2N nodes require special handling in 'PhaseCFG::schedule_late' to ensure the
2159 // register allocator does not move the CastI2N below a safepoint. This is necessary to avoid having the raw pointer span a safepoint,
2160 // making it opaque to the GC. Unlike CheckCastPPs, which need extra handling in 'Scheduling::ComputeRegisterAntidependencies' due to
2161 // scalarization, CastI2N nodes are always used by a load if scalarization happens which inherently keeps them pinned above the safepoint.
2162 return value;
2163 } else {
2164 // Make sure to zero unused bits in the 32-bit value
2165 return Compile::narrow_value(value_bt, value, nullptr, &igvn, true);
2166 }
2167 }
2168
2169 void StoreFlatNode::store(GraphKit* kit, Node* base, Node* ptr, InlineTypeNode* value, bool null_free, DecoratorSet decorators) {
2170 value = value->allocate_fields(kit);
2171 StoreFlatNode* store = new StoreFlatNode(null_free, decorators);
2172 store->init_req(TypeFunc::Control, kit->control());
2173 store->init_req(TypeFunc::I_O, kit->top());
2174 store->init_req(TypeFunc::Memory, kit->reset_memory());
2175 store->init_req(TypeFunc::FramePtr, kit->frameptr());
2176 store->init_req(TypeFunc::ReturnAdr, kit->top());
2177
2178 store->init_req(TypeFunc::Parms, base);
2179 store->init_req(TypeFunc::Parms + 1, ptr);
2180 store->init_req(TypeFunc::Parms + 2, value);
2181 kit->kill_dead_locals();
2182 kit->add_safepoint_edges(store);
2183 store = kit->gvn().transform(store)->as_StoreFlat();
2184 kit->record_for_igvn(store);
2185
2186 kit->set_control(kit->gvn().transform(new ProjNode(store, TypeFunc::Control)));
2187 kit->set_all_memory(kit->gvn().transform(new ProjNode(store, TypeFunc::Memory)));
2188 }
2189
2190 bool StoreFlatNode::expand_non_atomic(PhaseIterGVN& igvn) const {
2191 assert(igvn.delay_transform(), "transformation must be delayed");
2192 if ((_decorators & C2_MISMATCHED) != 0) {
2193 return false;
2194 }
2195
2196 GraphKit kit(this, igvn);
2197 Node* base = this->base();
2198 Node* ptr = this->ptr();
2199 InlineTypeNode* value = this->value();
2200
2201 ciInlineKlass* vk = igvn.type(value)->inline_klass();
2202 for (int i = 0; i < vk->nof_nonstatic_fields(); i++) {
2203 ciField* field = vk->nonstatic_field_at(i);
2204 Node* field_ptr = kit.basic_plus_adr(base, ptr, field->offset_in_bytes() - vk->payload_offset());
2205 const TypePtr* field_ptr_type = field_ptr->Value(&igvn)->is_ptr();
2206 igvn.set_type(field_ptr, field_ptr_type);
2207 Node* field_value = value->field_value_by_offset(field->offset_in_bytes(), true);
2208 kit.access_store_at(base, field_ptr, field_ptr_type, field_value, igvn.type(field_value), field->type()->basic_type(), _decorators);
2209 }
2210
2211 if (!_null_free) {
2212 Node* null_marker_ptr = kit.basic_plus_adr(base, ptr, vk->null_marker_offset_in_payload());
2213 const TypePtr* null_marker_ptr_type = null_marker_ptr->Value(&igvn)->is_ptr();
2214 igvn.set_type(null_marker_ptr, null_marker_ptr_type);
2215 Node* null_marker_value = value->get_null_marker();
2216 kit.access_store_at(base, null_marker_ptr, null_marker_ptr_type, null_marker_value, TypeInt::BOOL, T_BOOLEAN, _decorators);
2217 }
2218
2219 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
2220 if (old_ctrl != nullptr) {
2221 igvn.replace_node(old_ctrl, kit.control());
2222 }
2223 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
2224 Node* new_mem = kit.reset_memory();
2225 if (old_mem != nullptr) {
2226 igvn.replace_node(old_mem, new_mem);
2227 }
2228 return true;
2229 }
2230
2231 void StoreFlatNode::expand_atomic(PhaseIterGVN& igvn) const {
2232 // Convert to a payload value <= 64-bit and write atomically.
2233 // The payload might contain at most two oop fields that must be narrow because otherwise they would be 64-bit
2234 // in size and would then be written by a "normal" oop store. If the payload contains oops, its size is always
2235 // 64-bit because the next smaller (power-of-two) size would be 32-bit which could only hold one narrow oop that
2236 // would then be written by a normal narrow oop store. These properties are asserted in 'convert_to_payload'.
2237 assert(igvn.delay_transform(), "transformation must be delayed");
2238 GraphKit kit(this, igvn);
2239 Node* base = this->base();
2240 Node* ptr = this->ptr();
2241 InlineTypeNode* value = this->value();
2242
2243 int oop_off_1 = -1;
2244 int oop_off_2 = -1;
2245 Node* payload = convert_to_payload(igvn, kit.control(), value, _null_free, oop_off_1, oop_off_2);
2246
2247 ciInlineKlass* vk = igvn.type(value)->inline_klass();
2248 assert(oop_off_1 == -1 || oop_off_1 == 0 || oop_off_1 == 4, "invalid layout for %s, first oop at offset %d", vk->name()->as_utf8(), oop_off_1);
2249 assert(oop_off_2 == -1 || oop_off_2 == 4, "invalid layout for %s, second oop at offset %d", vk->name()->as_utf8(), oop_off_2);
2250 BasicType payload_bt = vk->atomic_size_to_basic_type(_null_free);
2251 kit.insert_mem_bar(Op_MemBarCPUOrder);
2252 if (!UseG1GC || oop_off_1 == -1) {
2253 // No oop fields or no late barrier expansion. Emit an atomic store of the payload and add GC barriers if needed.
2254 assert(oop_off_2 == -1 || !UseG1GC, "sanity");
2255 // ZGC does not support compressed oops, so only one oop can be in the payload which is written by a "normal" oop store.
2256 assert((oop_off_1 == -1 && oop_off_2 == -1) || !UseZGC, "ZGC does not support embedded oops in flat fields");
2257 kit.access_store_at(base, ptr, TypeRawPtr::BOTTOM, payload, Type::get_const_basic_type(payload_bt), payload_bt, _decorators | C2_MISMATCHED, true, value);
2258 } else {
2259 // Contains oops and requires late barrier expansion. Emit a special store node that allows to emit GC barriers in the backend.
2260 assert(UseG1GC, "Unexpected GC");
2261 assert(payload_bt == T_LONG, "Unexpected payload type");
2262 // If one oop, set the offset (if no offset is set, two oops are assumed by the backend)
2263 Node* oop_offset = (oop_off_2 == -1) ? igvn.intcon(oop_off_1) : nullptr;
2264 Node* mem = kit.reset_memory();
2265 kit.set_all_memory(mem);
2266 Node* store = igvn.transform(new StoreLSpecialNode(kit.control(), mem, ptr, TypeRawPtr::BOTTOM, payload, oop_offset, MemNode::unordered));
2267 kit.set_memory(store, TypeRawPtr::BOTTOM);
2268 }
2269 kit.insert_mem_bar(Op_MemBarCPUOrder);
2270
2271 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
2272 if (old_ctrl != nullptr) {
2273 igvn.replace_node(old_ctrl, kit.control());
2274 }
2275 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
2276 Node* new_mem = kit.reset_memory();
2277 if (old_mem != nullptr) {
2278 igvn.replace_node(old_mem, new_mem);
2279 }
2280 }
2281
2282 // Convert the field values to a payload value of type 'bt'
2283 Node* StoreFlatNode::convert_to_payload(PhaseIterGVN& igvn, Node* ctrl, InlineTypeNode* value, bool null_free, int& oop_off_1, int& oop_off_2) {
2284 ciInlineKlass* vk = igvn.type(value)->inline_klass();
2285 BasicType payload_bt = vk->atomic_size_to_basic_type(null_free);
2286 Node* payload = igvn.zerocon(payload_bt);
2287 if (!null_free) {
2288 // Set the null marker
2289 payload = set_payload_value(igvn, payload_bt, payload, T_BOOLEAN, value->get_null_marker(), vk->null_marker_offset_in_payload());
2290 }
2291
2292 // Iterate over the fields and add their values to the payload
2293 for (int i = 0; i < vk->nof_nonstatic_fields(); i++) {
2294 ciField* field = vk->nonstatic_field_at(i);
2295 Node* field_value = value->field_value_by_offset(field->offset_in_bytes(), true);
2296 ciType* field_klass = field->type();
2297 BasicType field_bt = field_klass->basic_type();
2298 int field_offset_in_payload = field->offset_in_bytes() - vk->payload_offset();
2299 if (!field_klass->is_primitive_type()) {
2300 // Narrow oop field
2301 assert(UseCompressedOops && payload_bt == T_LONG, "Naturally atomic");
2302 if (oop_off_1 == -1) {
2303 oop_off_1 = field_offset_in_payload;
2304 } else {
2305 assert(oop_off_2 == -1, "already set");
2306 oop_off_2 = field_offset_in_payload;
2307 }
2308
2309 const Type* val_type = Type::get_const_type(field_klass)->make_narrowoop();
2310 if (field_value->is_InlineType()) {
2311 assert(field_value->as_InlineType()->is_allocated(&igvn), "must be allocated");
2312 }
2313
2314 field_value = igvn.transform(new EncodePNode(field_value, val_type));
2315 field_value = igvn.transform(new CastP2XNode(ctrl, field_value));
2316 field_value = igvn.transform(new ConvL2INode(field_value));
2317 field_bt = T_INT;
2318 }
2319 payload = set_payload_value(igvn, payload_bt, payload, field_bt, field_value, field_offset_in_payload);
2320 }
2321
2322 return payload;
2323 }
2324
2325 Node* StoreFlatNode::set_payload_value(PhaseIterGVN& igvn, BasicType payload_bt, Node* payload, BasicType val_bt, Node* value, int offset) {
2326 assert((offset + type2aelembytes(val_bt)) <= type2aelembytes(payload_bt), "Value does not fit into payload");
2327
2328 // Make sure to zero unused bits in the 32-bit value
2329 if (val_bt == T_BYTE || val_bt == T_BOOLEAN) {
2330 value = igvn.transform(new AndINode(value, igvn.intcon(0xFF)));
2331 } else if (val_bt == T_CHAR || val_bt == T_SHORT) {
2332 value = igvn.transform(new AndINode(value, igvn.intcon(0xFFFF)));
2333 } else if (val_bt == T_FLOAT) {
2334 value = igvn.transform(new MoveF2INode(value));
2335 } else {
2336 assert(val_bt == T_INT, "Unsupported type: %s", type2name(val_bt));
2337 }
2338
2339 Node* shift_val = igvn.intcon(offset << LogBitsPerByte);
2340 if (payload_bt == T_LONG) {
2341 // Convert to long and remove the sign bit (the backend will fold this and emit a zero extend i2l)
2342 value = igvn.transform(new ConvI2LNode(value));
2343 value = igvn.transform(new AndLNode(value, igvn.longcon(0xFFFFFFFF)));
2344
2345 Node* shift_value = igvn.transform(new LShiftLNode(value, shift_val));
2346 payload = new OrLNode(shift_value, payload);
2347 } else {
2348 Node* shift_value = igvn.transform(new LShiftINode(value, shift_val));
2349 payload = new OrINode(shift_value, payload);
2350 }
2351 return igvn.transform(payload);
2352 }
2353
2354 const Type* LoadFlatNode::Value(PhaseGVN* phase) const {
2355 if (phase->type(in(TypeFunc::Control)) == Type::TOP || phase->type(in(TypeFunc::Memory)) == Type::TOP ||
2356 phase->type(base()) == Type::TOP || phase->type(ptr()) == Type::TOP) {
2357 return Type::TOP;
2358 }
2359 return bottom_type();
2360 }
2361
2362 const Type* StoreFlatNode::Value(PhaseGVN* phase) const {
2363 if (phase->type(in(TypeFunc::Control)) == Type::TOP || phase->type(in(TypeFunc::Memory)) == Type::TOP ||
2364 phase->type(base()) == Type::TOP || phase->type(ptr()) == Type::TOP || phase->type(value()) == Type::TOP) {
2365 return Type::TOP;
2366 }
2367 return bottom_type();
2368 }