1 /*
2 * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "ci/ciInlineKlass.hpp"
26 #include "gc/shared/barrierSet.hpp"
27 #include "gc/shared/c2/barrierSetC2.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "oops/accessDecorators.hpp"
30 #include "opto/addnode.hpp"
31 #include "opto/castnode.hpp"
32 #include "opto/compile.hpp"
33 #include "opto/convertnode.hpp"
34 #include "opto/graphKit.hpp"
35 #include "opto/inlinetypenode.hpp"
36 #include "opto/memnode.hpp"
37 #include "opto/movenode.hpp"
38 #include "opto/multnode.hpp"
39 #include "opto/narrowptrnode.hpp"
40 #include "opto/opcodes.hpp"
41 #include "opto/phaseX.hpp"
42 #include "opto/rootnode.hpp"
43 #include "opto/type.hpp"
44 #include "utilities/globalDefinitions.hpp"
45
46 // Clones the inline type to handle control flow merges involving multiple inline types.
47 // The inputs are replaced by PhiNodes to represent the merged values for the given region.
48 // init_with_top: input of phis above the returned InlineTypeNode are initialized to top.
49 InlineTypeNode* InlineTypeNode::clone_with_phis(PhaseGVN* gvn, Node* region, SafePointNode* map, bool is_non_null, bool init_with_top) {
50 InlineTypeNode* vt = clone_if_required(gvn, map);
51 const Type* t = Type::get_const_type(inline_klass());
52 gvn->set_type(vt, t);
53 vt->as_InlineType()->set_type(t);
54
55 Node* const top = gvn->C->top();
56
57 // Create a PhiNode for merging the oop values
58 PhiNode* oop = PhiNode::make(region, init_with_top ? top : vt->get_oop(), t);
59 gvn->set_type(oop, t);
60 gvn->record_for_igvn(oop);
61 vt->set_oop(*gvn, oop);
62
63 // Create a PhiNode for merging the is_buffered values
64 t = Type::get_const_basic_type(T_BOOLEAN);
65 Node* is_buffered_node = PhiNode::make(region, init_with_top ? top : vt->get_is_buffered(), t);
66 gvn->set_type(is_buffered_node, t);
67 gvn->record_for_igvn(is_buffered_node);
68 vt->set_req(IsBuffered, is_buffered_node);
69
70 // Create a PhiNode for merging the null_marker values
71 Node* null_marker_node;
72 if (is_non_null) {
73 null_marker_node = gvn->intcon(1);
74 } else {
75 t = Type::get_const_basic_type(T_BOOLEAN);
76 null_marker_node = PhiNode::make(region, init_with_top ? top : vt->get_null_marker(), t);
77 gvn->set_type(null_marker_node, t);
78 gvn->record_for_igvn(null_marker_node);
79 }
80 vt->set_req(NullMarker, null_marker_node);
81
82 // Create a PhiNode each for merging the field values
83 for (uint i = 0; i < vt->field_count(); ++i) {
84 ciType* type = vt->field(i)->type();
85 Node* value = vt->field_value(i);
86 // We limit scalarization for inline types with circular fields and can therefore observe nodes
87 // of the same type but with different scalarization depth during GVN. To avoid inconsistencies
88 // during merging, make sure that we only create Phis for fields that are guaranteed to be scalarized.
89 ciField* field = this->field(i);
90 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
91 bool no_circularity = !gvn->C->has_circular_inline_type() || field->is_flat();
92 if (type->is_inlinetype() && no_circularity) {
93 // Handle inline type fields recursively
94 value = value->as_InlineType()->clone_with_phis(gvn, region, map);
95 } else {
96 t = Type::get_const_type(type);
97 value = PhiNode::make(region, init_with_top ? top : value, t);
98 gvn->set_type(value, t);
99 gvn->record_for_igvn(value);
100 }
101 vt->set_field_value(i, value);
102 }
103 gvn->record_for_igvn(vt);
104 return vt;
105 }
106
107 // Checks if the inputs of the InlineTypeNode were replaced by PhiNodes
108 // for the given region (see InlineTypeNode::clone_with_phis).
109 bool InlineTypeNode::has_phi_inputs(Node* region) {
110 // Check oop input
111 bool result = get_oop()->is_Phi() && get_oop()->as_Phi()->region() == region;
112 #ifdef ASSERT
113 if (result) {
114 // Check all field value inputs for consistency
115 for (uint i = 0; i < field_count(); ++i) {
116 Node* n = field_value(i);
117 if (n->is_InlineType()) {
118 assert(n->as_InlineType()->has_phi_inputs(region), "inconsistent phi inputs");
119 } else {
120 assert(n->is_Phi() && n->as_Phi()->region() == region, "inconsistent phi inputs");
121 }
122 }
123 }
124 #endif
125 return result;
126 }
127
128 // Merges 'this' with 'other' by updating the input PhiNodes added by 'clone_with_phis'
129 InlineTypeNode* InlineTypeNode::merge_with(PhaseGVN* gvn, const InlineTypeNode* other, int phi_index, bool transform) {
130 assert(inline_klass() == other->inline_klass(), "Merging incompatible types");
131
132 // Merge oop inputs
133 PhiNode* phi = get_oop()->as_Phi();
134 phi->set_req(phi_index, other->get_oop());
135 if (transform) {
136 set_oop(*gvn, gvn->transform(phi));
137 }
138
139 // Merge is_buffered inputs
140 phi = get_is_buffered()->as_Phi();
141 phi->set_req(phi_index, other->get_is_buffered());
142 if (transform) {
143 set_req(IsBuffered, gvn->transform(phi));
144 }
145
146 // Merge null_marker inputs
147 Node* null_marker = get_null_marker();
148 if (null_marker->is_Phi()) {
149 phi = null_marker->as_Phi();
150 phi->set_req(phi_index, other->get_null_marker());
151 if (transform) {
152 set_req(NullMarker, gvn->transform(phi));
153 }
154 } else {
155 assert(null_marker->find_int_con(0) == 1, "only with a non null inline type");
156 }
157
158 // Merge field values
159 for (uint i = 0; i < field_count(); ++i) {
160 Node* val1 = field_value(i);
161 Node* val2 = other->field_value(i);
162 if (val1->is_InlineType()) {
163 if (val2->is_Phi()) {
164 val2 = gvn->transform(val2);
165 }
166 if (val2->is_top()) {
167 // The path where 'other' is used is dying. Therefore, we do not need to process the merge with 'other' further.
168 // The phi inputs of 'this' at 'phi_index' will eventually be removed.
169 break;
170 }
171 val1->as_InlineType()->merge_with(gvn, val2->as_InlineType(), phi_index, transform);
172 } else {
173 assert(val1->is_Phi(), "must be a phi node");
174 val1->set_req(phi_index, val2);
175 }
176 if (transform) {
177 set_field_value(i, gvn->transform(val1));
178 }
179 }
180 return this;
181 }
182
183 // Adds a new merge path to an inline type node with phi inputs
184 void InlineTypeNode::add_new_path(Node* region) {
185 assert(has_phi_inputs(region), "must have phi inputs");
186
187 PhiNode* phi = get_oop()->as_Phi();
188 phi->add_req(nullptr);
189 assert(phi->req() == region->req(), "must be same size as region");
190
191 phi = get_is_buffered()->as_Phi();
192 phi->add_req(nullptr);
193 assert(phi->req() == region->req(), "must be same size as region");
194
195 phi = get_null_marker()->as_Phi();
196 phi->add_req(nullptr);
197 assert(phi->req() == region->req(), "must be same size as region");
198
199 for (uint i = 0; i < field_count(); ++i) {
200 Node* val = field_value(i);
201 if (val->is_InlineType()) {
202 val->as_InlineType()->add_new_path(region);
203 } else {
204 val->as_Phi()->add_req(nullptr);
205 assert(val->req() == region->req(), "must be same size as region");
206 }
207 }
208 }
209
210 Node* InlineTypeNode::field_value(uint index) const {
211 assert(index < field_count(), "index out of bounds");
212 return in(Values + index);
213 }
214
215 // Get the value of the field at the given offset.
216 // If 'recursive' is true, flat inline type fields will be resolved recursively.
217 Node* InlineTypeNode::field_value_by_offset(int offset, bool recursive) const {
218 // Find the declared field which contains the field we are looking for
219 int index = inline_klass()->field_index_by_offset(offset);
220 Node* value = field_value(index);
221 assert(value != nullptr, "field value not found");
222 ciField* field = this->field(index);
223 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
224
225 if (!recursive || !field->is_flat() || value->is_top()) {
226 assert(offset == field->offset_in_bytes(), "offset mismatch");
227 return value;
228 }
229
230 // Flat inline type field
231 InlineTypeNode* vt = value->as_InlineType();
232 assert(field->is_flat(), "must be flat");
233 if (offset == field->null_marker_offset()) {
234 return vt->get_null_marker();
235 } else {
236 int sub_offset = offset - field->offset_in_bytes(); // Offset of the flattened field inside the declared field
237 sub_offset += vt->inline_klass()->payload_offset(); // Add header size
238 return vt->field_value_by_offset(sub_offset, recursive);
239 }
240 }
241
242 void InlineTypeNode::set_field_value(uint index, Node* value) {
243 assert(index < field_count(), "index out of bounds");
244 set_req(Values + index, value);
245 }
246
247 void InlineTypeNode::set_field_value_by_offset(int offset, Node* value) {
248 set_field_value(field_index(offset), value);
249 }
250
251 uint InlineTypeNode::field_index(int offset) const {
252 uint i = 0;
253 for (; i < field_count() && field(i)->offset_in_bytes() != offset; i++) { }
254 assert(i < field_count(), "field not found");
255 return i;
256 }
257
258 ciField* InlineTypeNode::field(uint index) const {
259 assert(index < field_count(), "index out of bounds");
260 return inline_klass()->declared_nonstatic_field_at(index);
261 }
262
263 uint InlineTypeNode::add_fields_to_safepoint(Unique_Node_List& worklist, SafePointNode* sfpt) {
264 uint cnt = 0;
265 for (uint i = 0; i < field_count(); ++i) {
266 Node* value = field_value(i);
267 ciField* field = this->field(i);
268 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
269 if (field->is_flat()) {
270 InlineTypeNode* vt = value->as_InlineType();
271 cnt += vt->add_fields_to_safepoint(worklist, sfpt);
272 if (!field->is_null_free()) {
273 // The null marker of a flat field is added right after we scalarize that field
274 sfpt->add_req(vt->get_null_marker());
275 cnt++;
276 }
277 continue;
278 }
279 if (value->is_InlineType()) {
280 // Add inline type to the worklist to process later
281 worklist.push(value);
282 }
283 sfpt->add_req(value);
284 cnt++;
285 }
286 return cnt;
287 }
288
289 void InlineTypeNode::make_scalar_in_safepoint(PhaseIterGVN* igvn, Unique_Node_List& worklist, SafePointNode* sfpt) {
290 JVMState* jvms = sfpt->jvms();
291 assert(jvms != nullptr, "missing JVMS");
292 uint first_ind = (sfpt->req() - jvms->scloff());
293
294 // Iterate over the inline type fields in order of increasing offset and add the
295 // field values to the safepoint. Nullable inline types have an null marker field that
296 // needs to be checked before using the field values.
297 sfpt->add_req(get_null_marker());
298 uint nfields = add_fields_to_safepoint(worklist, sfpt);
299 jvms->set_endoff(sfpt->req());
300 // Replace safepoint edge by SafePointScalarObjectNode
301 SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(type()->isa_instptr(),
302 nullptr,
303 first_ind,
304 sfpt->jvms()->depth(),
305 nfields);
306 sobj->init_req(0, igvn->C->root());
307 sobj = igvn->transform(sobj)->as_SafePointScalarObject();
308 igvn->rehash_node_delayed(sfpt);
309 for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) {
310 Node* debug = sfpt->in(i);
311 if (debug != nullptr && debug->uncast() == this) {
312 sfpt->set_req(i, sobj);
313 }
314 }
315 }
316
317 void InlineTypeNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop) {
318 // If the inline type has a constant or loaded oop, use the oop instead of scalarization
319 // in the safepoint to avoid keeping field loads live just for the debug info.
320 Node* oop = get_oop();
321 bool use_oop = false;
322 if (allow_oop && is_allocated(igvn) && oop->is_Phi()) {
323 Unique_Node_List worklist;
324 VectorSet visited;
325 visited.set(oop->_idx);
326 worklist.push(oop);
327 use_oop = true;
328 while (worklist.size() > 0 && use_oop) {
329 Node* n = worklist.pop();
330 for (uint i = 1; i < n->req(); i++) {
331 Node* in = n->in(i);
332 if (in->is_Phi() && !visited.test_set(in->_idx)) {
333 worklist.push(in);
334 } else if (!(in->is_Con() || in->is_Parm())) {
335 use_oop = false;
336 break;
337 }
338 }
339 }
340 } else {
341 use_oop = allow_oop && is_allocated(igvn) &&
342 (oop->is_Con() || oop->is_Parm() || oop->is_Load() || (oop->isa_DecodeN() && oop->in(1)->is_Load()));
343 }
344
345 ResourceMark rm;
346 Unique_Node_List safepoints;
347 Unique_Node_List vt_worklist;
348 Unique_Node_List worklist;
349 worklist.push(this);
350 while (worklist.size() > 0) {
351 Node* n = worklist.pop();
352 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
353 Node* use = n->fast_out(i);
354 if (use->is_SafePoint() && !use->is_CallLeaf() && (!use->is_Call() || use->as_Call()->has_debug_use(n))) {
355 safepoints.push(use);
356 } else if (use->is_ConstraintCast()) {
357 worklist.push(use);
358 }
359 }
360 }
361
362 // Process all safepoint uses and scalarize inline type
363 while (safepoints.size() > 0) {
364 SafePointNode* sfpt = safepoints.pop()->as_SafePoint();
365 if (use_oop) {
366 for (uint i = sfpt->jvms()->debug_start(); i < sfpt->jvms()->debug_end(); i++) {
367 Node* debug = sfpt->in(i);
368 if (debug != nullptr && debug->uncast() == this) {
369 sfpt->set_req(i, get_oop());
370 }
371 }
372 igvn->rehash_node_delayed(sfpt);
373 } else {
374 make_scalar_in_safepoint(igvn, vt_worklist, sfpt);
375 }
376 }
377 // Now scalarize non-flat fields
378 for (uint i = 0; i < vt_worklist.size(); ++i) {
379 InlineTypeNode* vt = vt_worklist.at(i)->isa_InlineType();
380 vt->make_scalar_in_safepoints(igvn);
381 }
382 if (outcnt() == 0) {
383 igvn->record_for_igvn(this);
384 }
385 }
386
387 // We limit scalarization for inline types with circular fields and can therefore observe nodes
388 // of the same type but with different scalarization depth during GVN. This method adjusts the
389 // scalarization depth to avoid inconsistencies during merging.
390 InlineTypeNode* InlineTypeNode::adjust_scalarization_depth(GraphKit* kit) {
391 if (!kit->C->has_circular_inline_type()) {
392 return this;
393 }
394 GrowableArray<ciType*> visited;
395 visited.push(inline_klass());
396 return adjust_scalarization_depth_impl(kit, visited);
397 }
398
399 InlineTypeNode* InlineTypeNode::adjust_scalarization_depth_impl(GraphKit* kit, GrowableArray<ciType*>& visited) {
400 InlineTypeNode* val = this;
401 for (uint i = 0; i < field_count(); ++i) {
402 Node* value = field_value(i);
403 Node* new_value = value;
404 ciField* field = this->field(i);
405 ciType* ft = field->type();
406 if (value->is_InlineType()) {
407 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
408 if (!field->is_flat() && visited.contains(ft)) {
409 new_value = value->as_InlineType()->buffer(kit)->get_oop();
410 } else {
411 int old_len = visited.length();
412 visited.push(ft);
413 new_value = value->as_InlineType()->adjust_scalarization_depth_impl(kit, visited);
414 visited.trunc_to(old_len);
415 }
416 } else if (ft->is_inlinetype() && !visited.contains(ft)) {
417 int old_len = visited.length();
418 visited.push(ft);
419 new_value = make_from_oop_impl(kit, value, ft->as_inline_klass(), visited);
420 visited.trunc_to(old_len);
421 }
422 if (value != new_value) {
423 if (val == this) {
424 val = clone_if_required(&kit->gvn(), kit->map());
425 }
426 val->set_field_value(i, new_value);
427 }
428 }
429 return (val == this) ? this : kit->gvn().transform(val)->as_InlineType();
430 }
431
432 void InlineTypeNode::load(GraphKit* kit, Node* base, Node* ptr, bool immutable_memory, bool trust_null_free_oop, DecoratorSet decorators, GrowableArray<ciType*>& visited) {
433 // Initialize the inline type by loading its field values from
434 // memory and adding the values as input edges to the node.
435 ciInlineKlass* vk = inline_klass();
436 for (uint i = 0; i < field_count(); ++i) {
437 ciField* field = this->field(i);
438 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
439 int field_off = field->offset_in_bytes() - vk->payload_offset();
440 Node* field_ptr = kit->basic_plus_adr(base, ptr, field_off);
441 Node* value = nullptr;
442 ciType* ft = field->type();
443 bool field_null_free = field->is_null_free();
444 if (field->is_flat()) {
445 // Recursively load the flat inline type field
446 ciInlineKlass* fvk = ft->as_inline_klass();
447 bool atomic = field->is_atomic();
448
449 int old_len = visited.length();
450 visited.push(ft);
451 value = make_from_flat_impl(kit, fvk, base, field_ptr, atomic, immutable_memory,
452 field_null_free, trust_null_free_oop && field_null_free, decorators, visited);
453 visited.trunc_to(old_len);
454 } else {
455 // Load field value from memory
456 BasicType bt = type2field[ft->basic_type()];
457 assert(is_java_primitive(bt) || field_ptr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
458 const Type* val_type = Type::get_const_type(ft);
459 if (trust_null_free_oop && field_null_free) {
460 val_type = val_type->join_speculative(TypePtr::NOTNULL);
461 }
462 const TypePtr* field_ptr_type = (decorators & C2_MISMATCHED) == 0 ? kit->gvn().type(field_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
463 value = kit->access_load_at(base, field_ptr, field_ptr_type, val_type, bt, decorators);
464 // Loading a non-flattened inline type from memory
465 if (visited.contains(ft)) {
466 kit->C->set_has_circular_inline_type(true);
467 } else if (ft->is_inlinetype()) {
468 int old_len = visited.length();
469 visited.push(ft);
470 value = make_from_oop_impl(kit, value, ft->as_inline_klass(), visited);
471 visited.trunc_to(old_len);
472 }
473 }
474 set_field_value(i, value);
475 }
476 }
477
478 void InlineTypeNode::store_flat(GraphKit* kit, Node* base, Node* ptr, bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators) {
479 ciInlineKlass* vk = inline_klass();
480 bool do_atomic = atomic;
481 // With immutable memory, a non-atomic load and an atomic load are the same
482 if (immutable_memory) {
483 do_atomic = false;
484 }
485 // If there is only one flattened field, a non-atomic load and an atomic load are the same
486 if (vk->is_naturally_atomic(null_free)) {
487 do_atomic = false;
488 }
489
490 if (!do_atomic) {
491 if (!null_free) {
492 int nm_offset = vk->null_marker_offset_in_payload();
493 Node* nm_ptr = kit->basic_plus_adr(base, ptr, nm_offset);
494 const TypePtr* nm_ptr_type = (decorators & C2_MISMATCHED) == 0 ? kit->gvn().type(nm_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
495 kit->access_store_at(base, nm_ptr, nm_ptr_type, get_null_marker(), TypeInt::BOOL, T_BOOLEAN, decorators);
496 }
497 store(kit, base, ptr, immutable_memory, decorators);
498 return;
499 }
500
501 StoreFlatNode::store(kit, base, ptr, this, null_free, decorators);
502 }
503
504 void InlineTypeNode::store_flat_array(GraphKit* kit, Node* base, Node* idx) {
505 PhaseGVN& gvn = kit->gvn();
506 DecoratorSet decorators = IN_HEAP | IS_ARRAY | MO_UNORDERED;
507 kit->C->set_flat_accesses();
508 ciInlineKlass* vk = inline_klass();
509 assert(vk->maybe_flat_in_array(), "element type %s cannot be flat in array", vk->name()->as_utf8());
510
511 RegionNode* region = new RegionNode(4);
512 gvn.set_type(region, Type::CONTROL);
513 kit->record_for_igvn(region);
514
515 Node* input_memory_state = kit->reset_memory();
516 kit->set_all_memory(input_memory_state);
517
518 PhiNode* mem = PhiNode::make(region, input_memory_state, Type::MEMORY, TypePtr::BOTTOM);
519 gvn.set_type(mem, Type::MEMORY);
520 kit->record_for_igvn(mem);
521
522 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO);
523 gvn.set_type(io, Type::ABIO);
524 kit->record_for_igvn(io);
525
526 Node* bol_null_free = kit->null_free_array_test(base); // Argument evaluation order is undefined in C++ and since this sets control, it needs to come first
527 IfNode* iff_null_free = kit->create_and_map_if(kit->control(), bol_null_free, PROB_FAIR, COUNT_UNKNOWN);
528
529 // Nullable
530 kit->set_control(kit->IfFalse(iff_null_free));
531 if (!kit->stopped()) {
532 assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8());
533 kit->set_all_memory(input_memory_state);
534 Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true);
535 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
536 store_flat(kit, cast, ptr, true, false, false, decorators);
537
538 region->init_req(1, kit->control());
539 mem->set_req(1, kit->reset_memory());
540 io->set_req(1, kit->i_o());
541 }
542
543 // Null-free
544 kit->set_control(kit->IfTrue(iff_null_free));
545 if (!kit->stopped()) {
546 kit->set_all_memory(input_memory_state);
547
548 Node* bol_atomic = kit->null_free_atomic_array_test(base, vk);
549 IfNode* iff_atomic = kit->create_and_map_if(kit->control(), bol_atomic, PROB_FAIR, COUNT_UNKNOWN);
550
551 // Atomic
552 kit->set_control(kit->IfTrue(iff_atomic));
553 if (!kit->stopped()) {
554 assert(vk->has_null_free_atomic_layout(), "element type %s does not have a null-free atomic flat layout", vk->name()->as_utf8());
555 kit->set_all_memory(input_memory_state);
556 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, true);
557 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
558 store_flat(kit, cast, ptr, true, false, true, decorators);
559
560 region->init_req(2, kit->control());
561 mem->set_req(2, kit->reset_memory());
562 io->set_req(2, kit->i_o());
563 }
564
565 // Non-atomic
566 kit->set_control(kit->IfFalse(iff_atomic));
567 if (!kit->stopped()) {
568 assert(vk->has_null_free_non_atomic_layout(), "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8());
569 kit->set_all_memory(input_memory_state);
570 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, false);
571 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
572 store_flat(kit, cast, ptr, false, false, true, decorators);
573
574 region->init_req(3, kit->control());
575 mem->set_req(3, kit->reset_memory());
576 io->set_req(3, kit->i_o());
577 }
578 }
579
580 kit->set_control(gvn.transform(region));
581 kit->set_all_memory(gvn.transform(mem));
582 kit->set_i_o(gvn.transform(io));
583 }
584
585 void InlineTypeNode::store(GraphKit* kit, Node* base, Node* ptr, bool immutable_memory, DecoratorSet decorators) const {
586 // Write field values to memory
587 ciInlineKlass* vk = inline_klass();
588 for (uint i = 0; i < field_count(); ++i) {
589 ciField* field = this->field(i);
590 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
591 int field_off = field->offset_in_bytes() - vk->payload_offset();
592 Node* field_val = field_value(i);
593 bool field_null_free = field->is_null_free();
594 ciType* ft = field->type();
595 Node* field_ptr = kit->basic_plus_adr(base, ptr, field_off);
596 if (field->is_flat()) {
597 // Recursively store the flat inline type field
598 ciInlineKlass* fvk = ft->as_inline_klass();
599 bool atomic = field->is_atomic();
600
601 field_val->as_InlineType()->store_flat(kit, base, field_ptr, atomic, immutable_memory, field_null_free, decorators);
602 } else {
603 // Store field value to memory
604 BasicType bt = type2field[ft->basic_type()];
605 const TypePtr* field_ptr_type = (decorators & C2_MISMATCHED) == 0 ? kit->gvn().type(field_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
606 const Type* val_type = Type::get_const_type(ft);
607 kit->access_store_at(base, field_ptr, field_ptr_type, field_val, val_type, bt, decorators);
608 }
609 }
610 }
611
612 // Adds a check between val1 and val2. Jumps to 'region' if check passes and optionally sets the corresponding phi input to false.
613 static void acmp_val_guard(PhaseIterGVN* igvn, RegionNode* region, Node* phi, Node** ctrl, BasicType bt, BoolTest::mask test, Node* val1, Node* val2) {
614 Node* cmp = nullptr;
615 switch (bt) {
616 case T_FLOAT:
617 val1 = igvn->register_new_node_with_optimizer(new MoveF2INode(val1));
618 val2 = igvn->register_new_node_with_optimizer(new MoveF2INode(val2));
619 // Fall-through to the int case
620 case T_BOOLEAN:
621 case T_CHAR:
622 case T_BYTE:
623 case T_SHORT:
624 case T_INT:
625 cmp = igvn->register_new_node_with_optimizer(new CmpINode(val1, val2));
626 break;
627 case T_DOUBLE:
628 val1 = igvn->register_new_node_with_optimizer(new MoveD2LNode(val1));
629 val2 = igvn->register_new_node_with_optimizer(new MoveD2LNode(val2));
630 // Fall-through to the long case
631 case T_LONG:
632 cmp = igvn->register_new_node_with_optimizer(new CmpLNode(val1, val2));
633 break;
634 default:
635 assert(is_reference_type(bt), "must be");
636 cmp = igvn->register_new_node_with_optimizer(new CmpPNode(val1, val2));
637 }
638 Node* bol = igvn->register_new_node_with_optimizer(new BoolNode(cmp, test));
639 IfNode* iff = igvn->register_new_node_with_optimizer(new IfNode(*ctrl, bol, PROB_MAX, COUNT_UNKNOWN))->as_If();
640 Node* if_f = igvn->register_new_node_with_optimizer(new IfFalseNode(iff));
641 Node* if_t = igvn->register_new_node_with_optimizer(new IfTrueNode(iff));
642
643 region->add_req(if_t);
644 if (phi != nullptr) {
645 phi->add_req(igvn->intcon(0));
646 }
647 *ctrl = if_f;
648 }
649
650 // Check if a substitutability check between 'this' and 'other' can be implemented in IR
651 bool InlineTypeNode::can_emit_substitutability_check(Node* other) const {
652 if (other != nullptr && other->is_InlineType() && bottom_type() != other->bottom_type()) {
653 // Different types, this is dead code because there's a check above that guarantees this.
654 return false;
655 }
656 for (uint i = 0; i < field_count(); i++) {
657 ciType* ft = field(i)->type();
658 Node* fv = field_value(i);
659 if (ft->is_inlinetype() && fv->is_InlineType()) {
660 // Check recursively
661 if (!fv->as_InlineType()->can_emit_substitutability_check(nullptr)){
662 return false;
663 }
664 } else if (ft->can_be_inline_klass()) {
665 // Comparing this field might require (another) substitutability check, bail out
666 return false;
667 }
668 }
669 return true;
670 }
671
672 // Emit IR to check substitutability between 'this' (left operand) and the value object referred to by 'other' (right operand).
673 // Parse-time checks guarantee that both operands have the same type. If 'other' is not an InlineTypeNode, we need to emit loads for the field values.
674 void InlineTypeNode::check_substitutability(PhaseIterGVN* igvn, RegionNode* region, Node* phi, Node** ctrl, Node* mem, Node* base, Node* other, bool flat) const {
675 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
676 DecoratorSet decorators = IN_HEAP | MO_UNORDERED | C2_READ_ACCESS | C2_CONTROL_DEPENDENT_LOAD;
677 MergeMemNode* local_mem = igvn->register_new_node_with_optimizer(MergeMemNode::make(mem))->as_MergeMem();
678
679 ciInlineKlass* vk = inline_klass();
680 for (uint i = 0; i < field_count(); i++) {
681 ciField* field = this->field(i);
682 int field_off = field->offset_in_bytes();
683 if (flat) {
684 // Flat access, no header
685 field_off -= vk->payload_offset();
686 }
687 Node* this_field = field_value(i);
688 ciType* ft = field->type();
689 BasicType bt = ft->basic_type();
690
691 Node* other_base = base;
692 Node* other_field = other;
693
694 // Get field value of the other operand
695 if (other->is_InlineType()) {
696 other_field = other->as_InlineType()->field_value(i);
697 other_base = nullptr;
698 } else {
699 // 'other' is an oop, compute address of the field
700 other_field = igvn->register_new_node_with_optimizer(AddPNode::make_with_base(base, other, igvn->MakeConX(field_off)));
701 if (field->is_flat()) {
702 // Flat field, load is handled recursively below
703 assert(this_field->is_InlineType(), "inconsistent field value");
704 } else {
705 // Non-flat field, load the field value and update the base because we are now operating on a different object
706 assert(is_java_primitive(bt) || other_field->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent field type");
707 C2AccessValuePtr addr(other_field, other_field->bottom_type()->is_ptr());
708 C2OptAccess access(*igvn, *ctrl, local_mem, decorators, bt, base, addr);
709 other_field = bs->load_at(access, Type::get_const_type(ft));
710 other_base = other_field;
711 }
712 }
713
714 if (this_field->is_InlineType()) {
715 RegionNode* done_region = new RegionNode(1);
716 ciField* field = this->field(i);
717 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
718 if (!field->is_null_free()) {
719 // Nullable field, check null marker before accessing the fields
720 if (field->is_flat()) {
721 // Flat field, check embedded null marker
722 Node* null_marker = nullptr;
723 if (other_field->is_InlineType()) {
724 // TODO 8350865 Should we add an IGVN optimization to fold null marker loads from InlineTypeNodes?
725 null_marker = other_field->as_InlineType()->get_null_marker();
726 } else {
727 Node* nm_offset = igvn->MakeConX(ft->as_inline_klass()->null_marker_offset_in_payload());
728 Node* nm_adr = igvn->register_new_node_with_optimizer(AddPNode::make_with_base(base, other_field, nm_offset));
729 C2AccessValuePtr addr(nm_adr, nm_adr->bottom_type()->is_ptr());
730 C2OptAccess access(*igvn, *ctrl, local_mem, decorators, T_BOOLEAN, base, addr);
731 null_marker = bs->load_at(access, TypeInt::BOOL);
732 }
733 // Return false if null markers are not equal
734 acmp_val_guard(igvn, region, phi, ctrl, T_INT, BoolTest::ne, this_field->as_InlineType()->get_null_marker(), null_marker);
735
736 // Null markers are equal. If both operands are null, skip the comparison of the fields.
737 acmp_val_guard(igvn, done_region, nullptr, ctrl, T_INT, BoolTest::eq, this_field->as_InlineType()->get_null_marker(), igvn->intcon(0));
738 } else {
739 // Non-flat field, check if oop is null
740
741 // Check if 'this' is null
742 RegionNode* not_null_region = new RegionNode(1);
743 acmp_val_guard(igvn, not_null_region, nullptr, ctrl, T_INT, BoolTest::ne, this_field->as_InlineType()->get_null_marker(), igvn->intcon(0));
744
745 // 'this' is null. If 'other' is non-null, return false.
746 acmp_val_guard(igvn, region, phi, ctrl, T_OBJECT, BoolTest::ne, other_field, igvn->zerocon(T_OBJECT));
747
748 // Both are null, skip comparing the fields
749 done_region->add_req(*ctrl);
750
751 // 'this' is not null. If 'other' is null, return false.
752 *ctrl = igvn->register_new_node_with_optimizer(not_null_region);
753 acmp_val_guard(igvn, region, phi, ctrl, T_OBJECT, BoolTest::eq, other_field, igvn->zerocon(T_OBJECT));
754 }
755 }
756 // Both operands are non-null, compare all the fields recursively
757 this_field->as_InlineType()->check_substitutability(igvn, region, phi, ctrl, mem, other_base, other_field, field->is_flat());
758
759 done_region->add_req(*ctrl);
760 *ctrl = igvn->register_new_node_with_optimizer(done_region);
761 } else {
762 assert(!ft->can_be_inline_klass(), "Needs substitutability test");
763 acmp_val_guard(igvn, region, phi, ctrl, bt, BoolTest::ne, this_field, other_field);
764 }
765 }
766 }
767
768 InlineTypeNode* InlineTypeNode::buffer(GraphKit* kit, bool safe_for_replace) {
769 if (kit->gvn().find_int_con(get_is_buffered(), 0) == 1) {
770 // Already buffered
771 return this;
772 }
773
774 // Check if inline type is already buffered
775 Node* not_buffered_ctl = kit->top();
776 Node* not_null_oop = kit->null_check_oop(get_oop(), ¬_buffered_ctl, /* never_see_null = */ false, safe_for_replace);
777 if (not_buffered_ctl->is_top()) {
778 // Already buffered
779 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map(), safe_for_replace);
780 vt->set_is_buffered(kit->gvn());
781 vt = kit->gvn().transform(vt)->as_InlineType();
782 if (safe_for_replace) {
783 kit->replace_in_map(this, vt);
784 }
785 return vt;
786 }
787 Node* buffered_ctl = kit->control();
788 kit->set_control(not_buffered_ctl);
789
790 // Inline type is not buffered, check if it is null.
791 Node* null_ctl = kit->top();
792 kit->null_check_common(get_null_marker(), T_INT, false, &null_ctl);
793 bool null_free = null_ctl->is_top();
794
795 RegionNode* region = new RegionNode(4);
796 PhiNode* oop = PhiNode::make(region, not_null_oop, type()->join_speculative(null_free ? TypePtr::NOTNULL : TypePtr::BOTTOM));
797
798 // InlineType is already buffered
799 region->init_req(1, buffered_ctl);
800 oop->init_req(1, not_null_oop);
801
802 // InlineType is null
803 region->init_req(2, null_ctl);
804 oop->init_req(2, kit->gvn().zerocon(T_OBJECT));
805
806 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO);
807 PhiNode* mem = PhiNode::make(region, kit->merged_memory(), Type::MEMORY, TypePtr::BOTTOM);
808
809 if (!kit->stopped()) {
810 assert(!is_allocated(&kit->gvn()), "already buffered");
811 PreserveJVMState pjvms(kit);
812 ciInlineKlass* vk = inline_klass();
813 // Allocate and initialize buffer, re-execute on deoptimization.
814 kit->jvms()->set_bci(kit->bci());
815 kit->jvms()->set_should_reexecute(true);
816 kit->kill_dead_locals();
817 Node* klass_node = kit->makecon(TypeKlassPtr::make(vk));
818 Node* alloc_oop = kit->new_instance(klass_node, nullptr, nullptr, /* deoptimize_on_exception */ true, this);
819 Node* payload_alloc_oop = kit->basic_plus_adr(alloc_oop, vk->payload_offset());
820 store(kit, alloc_oop, payload_alloc_oop, true, IN_HEAP | MO_UNORDERED | C2_TIGHTLY_COUPLED_ALLOC);
821
822 // Do not let stores that initialize this buffer be reordered with a subsequent
823 // store that would make this buffer accessible by other threads.
824 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_oop);
825 assert(alloc != nullptr, "must have an allocation node");
826 kit->insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress));
827 oop->init_req(3, alloc_oop);
828 region->init_req(3, kit->control());
829 io ->init_req(3, kit->i_o());
830 mem ->init_req(3, kit->merged_memory());
831 }
832
833 // Update GraphKit
834 kit->set_control(kit->gvn().transform(region));
835 kit->set_i_o(kit->gvn().transform(io));
836 kit->set_all_memory(kit->gvn().transform(mem));
837 kit->record_for_igvn(region);
838 kit->record_for_igvn(oop);
839 kit->record_for_igvn(io);
840 kit->record_for_igvn(mem);
841
842 // Use cloned InlineTypeNode to propagate oop from now on
843 Node* res_oop = kit->gvn().transform(oop);
844 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map(), safe_for_replace);
845 vt->set_oop(kit->gvn(), res_oop);
846 vt->set_is_buffered(kit->gvn());
847 vt = kit->gvn().transform(vt)->as_InlineType();
848 kit->record_for_igvn(vt);
849 if (safe_for_replace) {
850 kit->replace_in_map(this, vt);
851 }
852 // InlineTypeNode::remove_redundant_allocations piggybacks on split if.
853 // Make sure it gets a chance to remove this allocation.
854 kit->C->set_has_split_ifs(true);
855 return vt;
856 }
857
858 bool InlineTypeNode::is_allocated(PhaseGVN* phase) const {
859 if (phase->find_int_con(get_is_buffered(), 0) == 1) {
860 return true;
861 }
862 Node* oop = get_oop();
863 const Type* oop_type = (phase != nullptr) ? phase->type(oop) : oop->bottom_type();
864 return !oop_type->maybe_null();
865 }
866
867 static void replace_proj(Compile* C, CallNode* call, uint& proj_idx, Node* value, BasicType bt) {
868 ProjNode* pn = call->proj_out_or_null(proj_idx);
869 if (pn != nullptr) {
870 C->gvn_replace_by(pn, value);
871 C->initial_gvn()->hash_delete(pn);
872 pn->set_req(0, C->top());
873 }
874 proj_idx += type2size[bt];
875 }
876
877 // When a call returns multiple values, it has several result
878 // projections, one per field. Replacing the result of the call by an
879 // inline type node (after late inlining) requires that for each result
880 // projection, we find the corresponding inline type field.
881 void InlineTypeNode::replace_call_results(GraphKit* kit, CallNode* call, Compile* C) {
882 uint proj_idx = TypeFunc::Parms;
883 // Replace oop projection
884 replace_proj(C, call, proj_idx, get_oop(), T_OBJECT);
885 // Replace field projections
886 replace_field_projs(C, call, proj_idx);
887 // Replace null_marker projection
888 replace_proj(C, call, proj_idx, get_null_marker(), T_BOOLEAN);
889 assert(proj_idx == call->tf()->range_cc()->cnt(), "missed a projection");
890 }
891
892 void InlineTypeNode::replace_field_projs(Compile* C, CallNode* call, uint& proj_idx) {
893 for (uint i = 0; i < field_count(); ++i) {
894 Node* value = field_value(i);
895 ciField* field = this->field(i);
896 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
897 if (field->is_flat()) {
898 InlineTypeNode* vt = value->as_InlineType();
899 // Replace field projections for flat field
900 vt->replace_field_projs(C, call, proj_idx);
901 if (!field->is_null_free()) {
902 // Replace null_marker projection for nullable field
903 replace_proj(C, call, proj_idx, vt->get_null_marker(), T_BOOLEAN);
904 }
905 continue;
906 }
907 // Replace projection for field value
908 replace_proj(C, call, proj_idx, value, field->type()->basic_type());
909 }
910 }
911
912 InlineTypeNode* InlineTypeNode::allocate_fields(GraphKit* kit) {
913 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map());
914 for (uint i = 0; i < field_count(); i++) {
915 Node* value = field_value(i);
916 ciField* field = this->field(i);
917 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
918 if (field->is_flat()) {
919 // Flat inline type field
920 vt->set_field_value(i, value->as_InlineType()->allocate_fields(kit));
921 } else if (value->is_InlineType()) {
922 // Non-flat inline type field
923 vt->set_field_value(i, value->as_InlineType()->buffer(kit));
924 }
925 }
926 vt = kit->gvn().transform(vt)->as_InlineType();
927 kit->replace_in_map(this, vt);
928 return vt;
929 }
930
931 // Replace a buffer allocation by a dominating allocation
932 static void replace_allocation(PhaseIterGVN* igvn, Node* res, Node* dom) {
933 // Remove initializing stores and GC barriers
934 for (DUIterator_Fast imax, i = res->fast_outs(imax); i < imax; i++) {
935 Node* use = res->fast_out(i);
936 if (use->is_AddP()) {
937 for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) {
938 Node* store = use->fast_out(j)->isa_Store();
939 if (store != nullptr) {
940 igvn->rehash_node_delayed(store);
941 igvn->replace_in_uses(store, store->in(MemNode::Memory));
942 }
943 }
944 } else if (use->Opcode() == Op_CastP2X) {
945 if (UseG1GC && use->find_out_with(Op_XorX)->in(1) != use) {
946 // The G1 pre-barrier uses a CastP2X both for the pointer of the object
947 // we store into, as well as the value we are storing. Skip if this is a
948 // barrier for storing 'res' into another object.
949 continue;
950 }
951 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
952 bs->eliminate_gc_barrier(igvn, use);
953 --i; --imax;
954 }
955 }
956 igvn->replace_node(res, dom);
957 }
958
959 Node* InlineTypeNode::Ideal(PhaseGVN* phase, bool can_reshape) {
960 Node* oop = get_oop();
961 Node* is_buffered = get_is_buffered();
962
963 if (oop->isa_InlineType() && !phase->type(oop)->maybe_null()) {
964 InlineTypeNode* vtptr = oop->as_InlineType();
965 set_oop(*phase, vtptr->get_oop());
966 set_is_buffered(*phase);
967 set_null_marker(*phase);
968 for (uint i = Values; i < vtptr->req(); ++i) {
969 set_req(i, vtptr->in(i));
970 }
971 return this;
972 }
973
974 // Use base oop if fields are loaded from memory, don't do so if base is the CheckCastPP of an
975 // allocation because the only case we load from a naked CheckCastPP is when we exit a
976 // constructor of an inline type and we want to relinquish the larval oop there. This has a
977 // couple of benefits:
978 // - The allocation is likely to be elided earlier if it is not an input of an InlineTypeNode.
979 // - The InlineTypeNode without an allocation input is more likely to be GVN-ed. This may emerge
980 // when we try to clone a value object.
981 // - The buffering, if needed, is delayed until it is required. This new allocation, since it is
982 // created from an InlineTypeNode, is recognized as not having a unique identity and in the
983 // future, we can move them around more freely such as hoisting out of loops. This is not true
984 // for the old allocation since larval value objects do have unique identities.
985 Node* base = is_loaded(phase);
986 if (base != nullptr && !base->is_InlineType() && !phase->type(base)->maybe_null() && phase->C->allow_macro_nodes() && AllocateNode::Ideal_allocation(base) == nullptr) {
987 if (oop != base || phase->type(is_buffered) != TypeInt::ONE) {
988 set_oop(*phase, base);
989 set_is_buffered(*phase);
990 return this;
991 }
992 }
993
994 if (can_reshape) {
995 PhaseIterGVN* igvn = phase->is_IterGVN();
996 if (is_allocated(phase)) {
997 // Search for and remove re-allocations of this inline type. Ignore scalar replaceable ones,
998 // they will be removed anyway and changing the memory chain will confuse other optimizations.
999 // This can happen with late inlining when we first allocate an inline type argument
1000 // but later decide to inline the call after the callee code also triggered allocation.
1001 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1002 AllocateNode* alloc = fast_out(i)->isa_Allocate();
1003 if (alloc != nullptr && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable) {
1004 // Found a re-allocation
1005 Node* res = alloc->result_cast();
1006 if (res != nullptr && res->is_CheckCastPP()) {
1007 // Replace allocation by oop and unlink AllocateNode
1008 replace_allocation(igvn, res, oop);
1009 igvn->replace_input_of(alloc, AllocateNode::InlineType, igvn->C->top());
1010 --i; --imax;
1011 }
1012 }
1013 }
1014 }
1015 }
1016
1017 return nullptr;
1018 }
1019
1020 InlineTypeNode* InlineTypeNode::make_uninitialized(PhaseGVN& gvn, ciInlineKlass* vk, bool null_free) {
1021 // Create a new InlineTypeNode with uninitialized values and nullptr oop
1022 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT), null_free);
1023 vt->set_is_buffered(gvn, false);
1024 vt->set_null_marker(gvn);
1025 return vt;
1026 }
1027
1028 InlineTypeNode* InlineTypeNode::make_all_zero(PhaseGVN& gvn, ciInlineKlass* vk) {
1029 GrowableArray<ciType*> visited;
1030 visited.push(vk);
1031 return make_all_zero_impl(gvn, vk, visited);
1032 }
1033
1034 InlineTypeNode* InlineTypeNode::make_all_zero_impl(PhaseGVN& gvn, ciInlineKlass* vk, GrowableArray<ciType*>& visited) {
1035 // Create a new InlineTypeNode initialized with all zero
1036 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT), /* null_free= */ true);
1037 vt->set_is_buffered(gvn, false);
1038 vt->set_null_marker(gvn);
1039 for (uint i = 0; i < vt->field_count(); ++i) {
1040 ciField* field = vt->field(i);
1041 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1042 ciType* ft = field->type();
1043 Node* value = gvn.zerocon(ft->basic_type());
1044 if (!field->is_flat() && visited.contains(ft)) {
1045 gvn.C->set_has_circular_inline_type(true);
1046 } else if (ft->is_inlinetype()) {
1047 int old_len = visited.length();
1048 visited.push(ft);
1049 ciInlineKlass* vk = ft->as_inline_klass();
1050 if (field->is_null_free()) {
1051 value = make_all_zero_impl(gvn, vk, visited);
1052 } else {
1053 value = make_null_impl(gvn, vk, visited);
1054 }
1055 visited.trunc_to(old_len);
1056 }
1057 vt->set_field_value(i, value);
1058 }
1059 vt = gvn.transform(vt)->as_InlineType();
1060 assert(vt->is_all_zero(&gvn), "must be the all-zero inline type");
1061 return vt;
1062 }
1063
1064 bool InlineTypeNode::is_all_zero(PhaseGVN* gvn, bool flat) const {
1065 const TypeInt* tinit = gvn->type(get_null_marker())->isa_int();
1066 if (tinit == nullptr || !tinit->is_con(1)) {
1067 return false; // May be null
1068 }
1069 for (uint i = 0; i < field_count(); ++i) {
1070 Node* value = field_value(i);
1071 ciField* field = this->field(i);
1072 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1073 if (field->is_null_free()) {
1074 // Null-free value class field must have the all-zero value. If 'flat' is set,
1075 // reject non-flat fields because they need to be initialized with an oop to a buffer.
1076 if (!value->is_InlineType() || !value->as_InlineType()->is_all_zero(gvn) || (flat && !field->is_flat())) {
1077 return false;
1078 }
1079 continue;
1080 } else if (value->is_InlineType()) {
1081 // Nullable value class field must be null
1082 tinit = gvn->type(value->as_InlineType()->get_null_marker())->isa_int();
1083 if (tinit != nullptr && tinit->is_con(0)) {
1084 continue;
1085 }
1086 return false;
1087 } else if (!gvn->type(value)->is_zero_type()) {
1088 return false;
1089 }
1090 }
1091 return true;
1092 }
1093
1094 InlineTypeNode* InlineTypeNode::make_from_oop(GraphKit* kit, Node* oop, ciInlineKlass* vk) {
1095 GrowableArray<ciType*> visited;
1096 visited.push(vk);
1097 return make_from_oop_impl(kit, oop, vk, visited);
1098 }
1099
1100 InlineTypeNode* InlineTypeNode::make_from_oop_impl(GraphKit* kit, Node* oop, ciInlineKlass* vk, GrowableArray<ciType*>& visited) {
1101 PhaseGVN& gvn = kit->gvn();
1102
1103 // Create and initialize an InlineTypeNode by loading all field
1104 // values from a heap-allocated version and also save the oop.
1105 InlineTypeNode* vt = nullptr;
1106
1107 if (oop->isa_InlineType()) {
1108 return oop->as_InlineType();
1109 }
1110
1111 if (gvn.type(oop)->maybe_null()) {
1112 // Add a null check because the oop may be null
1113 Node* null_ctl = kit->top();
1114 Node* not_null_oop = kit->null_check_oop(oop, &null_ctl);
1115 if (kit->stopped()) {
1116 // Constant null
1117 kit->set_control(null_ctl);
1118 vt = make_null_impl(gvn, vk, visited);
1119 kit->record_for_igvn(vt);
1120 return vt;
1121 }
1122 vt = new InlineTypeNode(vk, not_null_oop, /* null_free= */ false);
1123 vt->set_is_buffered(gvn);
1124 vt->set_null_marker(gvn);
1125 Node* payload_ptr = kit->basic_plus_adr(not_null_oop, vk->payload_offset());
1126 vt->load(kit, not_null_oop, payload_ptr, true, true, IN_HEAP | MO_UNORDERED, visited);
1127
1128 if (null_ctl != kit->top()) {
1129 InlineTypeNode* null_vt = make_null_impl(gvn, vk, visited);
1130 Node* region = new RegionNode(3);
1131 region->init_req(1, kit->control());
1132 region->init_req(2, null_ctl);
1133 vt = vt->clone_with_phis(&gvn, region, kit->map());
1134 vt->merge_with(&gvn, null_vt, 2, true);
1135 vt->set_oop(gvn, oop);
1136 kit->set_control(gvn.transform(region));
1137 }
1138 } else {
1139 // Oop can never be null
1140 vt = new InlineTypeNode(vk, oop, /* null_free= */ true);
1141 Node* init_ctl = kit->control();
1142 vt->set_is_buffered(gvn);
1143 vt->set_null_marker(gvn);
1144 Node* payload_ptr = kit->basic_plus_adr(oop, vk->payload_offset());
1145 vt->load(kit, oop, payload_ptr, true, true, IN_HEAP | MO_UNORDERED, visited);
1146 // TODO 8284443
1147 // assert(!null_free || vt->as_InlineType()->is_all_zero(&gvn) || init_ctl != kit->control() || !gvn.type(oop)->is_inlinetypeptr() || oop->is_Con() || oop->Opcode() == Op_InlineType ||
1148 // AllocateNode::Ideal_allocation(oop, &gvn) != nullptr || vt->as_InlineType()->is_loaded(&gvn) == oop, "inline type should be loaded");
1149 }
1150 assert(vt->is_allocated(&gvn), "inline type should be allocated");
1151 kit->record_for_igvn(vt);
1152 return gvn.transform(vt)->as_InlineType();
1153 }
1154
1155 InlineTypeNode* InlineTypeNode::make_from_flat(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr,
1156 bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators) {
1157 GrowableArray<ciType*> visited;
1158 visited.push(vk);
1159 return make_from_flat_impl(kit, vk, base, ptr, atomic, immutable_memory, null_free, null_free, decorators, visited);
1160 }
1161
1162 // GraphKit wrapper for the 'make_from_flat' method
1163 InlineTypeNode* InlineTypeNode::make_from_flat_impl(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr, bool atomic, bool immutable_memory,
1164 bool null_free, bool trust_null_free_oop, DecoratorSet decorators, GrowableArray<ciType*>& visited) {
1165 assert(null_free || !trust_null_free_oop, "cannot trust null-free oop when the holder object is not null-free");
1166 PhaseGVN& gvn = kit->gvn();
1167 bool do_atomic = atomic;
1168 // With immutable memory, a non-atomic load and an atomic load are the same
1169 if (immutable_memory) {
1170 do_atomic = false;
1171 }
1172 // If there is only one flattened field, a non-atomic load and an atomic load are the same
1173 if (vk->is_naturally_atomic(null_free)) {
1174 do_atomic = false;
1175 }
1176
1177 if (!do_atomic) {
1178 InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk, null_free);
1179 if (!null_free) {
1180 int nm_offset = vk->null_marker_offset_in_payload();
1181 Node* nm_ptr = kit->basic_plus_adr(base, ptr, nm_offset);
1182 const TypePtr* nm_ptr_type = (decorators & C2_MISMATCHED) == 0 ? gvn.type(nm_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
1183 Node* nm_value = kit->access_load_at(base, nm_ptr, nm_ptr_type, TypeInt::BOOL, T_BOOLEAN, decorators);
1184 vt->set_req(NullMarker, nm_value);
1185 }
1186
1187 vt->load(kit, base, ptr, immutable_memory, trust_null_free_oop, decorators, visited);
1188 return gvn.transform(vt)->as_InlineType();
1189 }
1190
1191 assert(!immutable_memory, "immutable memory does not need explicit atomic access");
1192 return LoadFlatNode::load(kit, vk, base, ptr, null_free, trust_null_free_oop, decorators);
1193 }
1194
1195 InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* idx) {
1196 assert(vk->maybe_flat_in_array(), "element type %s cannot be flat in array", vk->name()->as_utf8());
1197 PhaseGVN& gvn = kit->gvn();
1198 DecoratorSet decorators = IN_HEAP | IS_ARRAY | MO_UNORDERED | C2_CONTROL_DEPENDENT_LOAD;
1199 kit->C->set_flat_accesses();
1200 InlineTypeNode* vt_nullable = nullptr;
1201 InlineTypeNode* vt_null_free = nullptr;
1202 InlineTypeNode* vt_non_atomic = nullptr;
1203
1204 RegionNode* region = new RegionNode(4);
1205 gvn.set_type(region, Type::CONTROL);
1206 kit->record_for_igvn(region);
1207
1208 Node* input_memory_state = kit->reset_memory();
1209 kit->set_all_memory(input_memory_state);
1210
1211 PhiNode* mem = PhiNode::make(region, input_memory_state, Type::MEMORY, TypePtr::BOTTOM);
1212 gvn.set_type(mem, Type::MEMORY);
1213 kit->record_for_igvn(mem);
1214
1215 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO);
1216 gvn.set_type(io, Type::ABIO);
1217 kit->record_for_igvn(io);
1218
1219 Node* bol_null_free = kit->null_free_array_test(base); // Argument evaluation order is undefined in C++ and since this sets control, it needs to come first
1220 IfNode* iff_null_free = kit->create_and_map_if(kit->control(), bol_null_free, PROB_FAIR, COUNT_UNKNOWN);
1221
1222 // Nullable
1223 kit->set_control(kit->IfFalse(iff_null_free));
1224 if (!kit->stopped()) {
1225 assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8());
1226 kit->set_all_memory(input_memory_state);
1227 Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true);
1228 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
1229 vt_nullable = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, false, decorators);
1230
1231 region->init_req(1, kit->control());
1232 mem->set_req(1, kit->reset_memory());
1233 io->set_req(1, kit->i_o());
1234 }
1235
1236 // Null-free
1237 kit->set_control(kit->IfTrue(iff_null_free));
1238 if (!kit->stopped()) {
1239 kit->set_all_memory(input_memory_state);
1240
1241 Node* bol_atomic = kit->null_free_atomic_array_test(base, vk);
1242 IfNode* iff_atomic = kit->create_and_map_if(kit->control(), bol_atomic, PROB_FAIR, COUNT_UNKNOWN);
1243
1244 // Atomic
1245 kit->set_control(kit->IfTrue(iff_atomic));
1246 if (!kit->stopped()) {
1247 assert(vk->has_null_free_atomic_layout(), "element type %s does not have a null-free atomic flat layout", vk->name()->as_utf8());
1248 kit->set_all_memory(input_memory_state);
1249 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, true);
1250 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
1251 vt_null_free = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, true, decorators);
1252
1253 region->init_req(2, kit->control());
1254 mem->set_req(2, kit->reset_memory());
1255 io->set_req(2, kit->i_o());
1256 }
1257
1258 // Non-Atomic
1259 kit->set_control(kit->IfFalse(iff_atomic));
1260 if (!kit->stopped()) {
1261 assert(vk->has_null_free_non_atomic_layout(), "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8());
1262 kit->set_all_memory(input_memory_state);
1263 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, false);
1264 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
1265 vt_non_atomic = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, false, false, true, decorators);
1266
1267 region->init_req(3, kit->control());
1268 mem->set_req(3, kit->reset_memory());
1269 io->set_req(3, kit->i_o());
1270 }
1271 }
1272
1273 InlineTypeNode* vt = nullptr;
1274 if (vt_nullable == nullptr && vt_null_free == nullptr && vt_non_atomic == nullptr) {
1275 // All paths are dead
1276 vt = make_null(gvn, vk);
1277 } else if (vt_nullable == nullptr && vt_null_free == nullptr) {
1278 vt = vt_non_atomic;
1279 } else if (vt_nullable == nullptr && vt_non_atomic == nullptr) {
1280 vt = vt_null_free;
1281 } else if (vt_null_free == nullptr && vt_non_atomic == nullptr) {
1282 vt = vt_nullable;
1283 }
1284 if (vt != nullptr) {
1285 kit->set_control(kit->gvn().transform(region));
1286 kit->set_all_memory(kit->gvn().transform(mem));
1287 kit->set_i_o(kit->gvn().transform(io));
1288 return vt;
1289 }
1290
1291 InlineTypeNode* zero = InlineTypeNode::make_null(gvn, vk);
1292 vt = zero->clone_with_phis(&gvn, region);
1293 if (vt_nullable != nullptr) {
1294 vt = vt->merge_with(&gvn, vt_nullable, 1, false);
1295 }
1296 if (vt_null_free != nullptr) {
1297 vt = vt->merge_with(&gvn, vt_null_free, 2, false);
1298 }
1299 if (vt_non_atomic != nullptr) {
1300 vt = vt->merge_with(&gvn, vt_non_atomic, 3, false);
1301 }
1302
1303 kit->set_control(kit->gvn().transform(region));
1304 kit->set_all_memory(kit->gvn().transform(mem));
1305 kit->set_i_o(kit->gvn().transform(io));
1306 return gvn.transform(vt)->as_InlineType();
1307 }
1308
1309 InlineTypeNode* InlineTypeNode::make_from_multi(GraphKit* kit, MultiNode* multi, ciInlineKlass* vk, uint& base_input, bool in, bool null_free) {
1310 InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk, null_free);
1311 if (!in || multi->is_Start()) {
1312 // Keep track of the oop. The returned inline type might already be buffered.
1313 Node* oop = nullptr;
1314 if (multi->is_Start()) {
1315 oop = kit->gvn().transform(new ParmNode(multi->as_Start(), base_input++));
1316 } else {
1317 oop = kit->gvn().transform(new ProjNode(multi, base_input++));
1318 }
1319 vt->set_oop(kit->gvn(), oop);
1320 } else {
1321 Node* oop = multi->as_Call()->in(base_input++);
1322 vt->set_oop(kit->gvn(), oop);
1323 }
1324 GrowableArray<ciType*> visited;
1325 visited.push(vk);
1326 vt->initialize_fields(kit, multi, base_input, in, null_free, nullptr, visited);
1327 return kit->gvn().transform(vt)->as_InlineType();
1328 }
1329
1330 Node* InlineTypeNode::is_loaded(PhaseGVN* phase, ciInlineKlass* vk, Node* base, int holder_offset) {
1331 if (vk == nullptr) {
1332 vk = inline_klass();
1333 }
1334 for (uint i = 0; i < field_count(); ++i) {
1335 ciField* field = this->field(i);
1336 int offset = holder_offset + field->offset_in_bytes();
1337 Node* value = field_value(i);
1338 if (value->is_InlineType()) {
1339 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1340 InlineTypeNode* vt = value->as_InlineType();
1341 if (vt->type()->inline_klass()->is_empty()) {
1342 continue;
1343 } else if (field->is_flat() && vt->is_InlineType()) {
1344 // Check inline type field load recursively
1345 base = vt->as_InlineType()->is_loaded(phase, vk, base, offset - vt->type()->inline_klass()->payload_offset());
1346 if (base == nullptr) {
1347 return nullptr;
1348 }
1349 continue;
1350 } else {
1351 value = vt->get_oop();
1352 if (value->Opcode() == Op_CastPP) {
1353 // Skip CastPP
1354 value = value->in(1);
1355 }
1356 }
1357 }
1358 if (value->isa_DecodeN()) {
1359 // Skip DecodeN
1360 value = value->in(1);
1361 }
1362 if (value->isa_Load()) {
1363 // Check if base and offset of field load matches inline type layout
1364 intptr_t loffset = 0;
1365 Node* lbase = AddPNode::Ideal_base_and_offset(value->in(MemNode::Address), phase, loffset);
1366 if (lbase == nullptr || (lbase != base && base != nullptr) || loffset != offset) {
1367 return nullptr;
1368 } else if (base == nullptr) {
1369 // Set base and check if pointer type matches
1370 base = lbase;
1371 const TypeInstPtr* vtptr = phase->type(base)->isa_instptr();
1372 if (vtptr == nullptr || !vtptr->instance_klass()->equals(vk)) {
1373 return nullptr;
1374 }
1375 }
1376 } else {
1377 return nullptr;
1378 }
1379 }
1380 return base;
1381 }
1382
1383 Node* InlineTypeNode::tagged_klass(ciInlineKlass* vk, PhaseGVN& gvn) {
1384 const TypeKlassPtr* tk = TypeKlassPtr::make(vk);
1385 intptr_t bits = tk->get_con();
1386 set_nth_bit(bits, 0);
1387 return gvn.longcon((jlong)bits);
1388 }
1389
1390 void InlineTypeNode::pass_fields(GraphKit* kit, Node* n, uint& base_input, bool in, bool null_free, bool root) {
1391 if (root) {
1392 n->init_req(base_input++, get_oop());
1393 }
1394 if (!null_free && in) {
1395 n->init_req(base_input++, get_null_marker());
1396 }
1397 for (uint i = 0; i < field_count(); i++) {
1398 Node* arg = field_value(i);
1399 ciField* field = this->field(i);
1400 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1401 if (field->is_flat()) {
1402 // Flat inline type field
1403 arg->as_InlineType()->pass_fields(kit, n, base_input, in);
1404 if (!field->is_null_free()) {
1405 assert(field->null_marker_offset() != -1, "inconsistency");
1406 n->init_req(base_input++, arg->as_InlineType()->get_null_marker());
1407 }
1408 } else {
1409 if (arg->is_InlineType()) {
1410 // Non-flat inline type field
1411 InlineTypeNode* vt = arg->as_InlineType();
1412 assert(n->Opcode() != Op_Return || vt->is_allocated(&kit->gvn()), "inline type field should be allocated on return");
1413 arg = vt->buffer(kit);
1414 }
1415 // Initialize call/return arguments
1416 n->init_req(base_input++, arg);
1417 if (field->type()->size() == 2) {
1418 n->init_req(base_input++, kit->top());
1419 }
1420 }
1421 }
1422 // The last argument is used to pass the null marker to compiled code and not required here.
1423 if (!null_free && !in) {
1424 n->init_req(base_input++, kit->top());
1425 }
1426 }
1427
1428 void InlineTypeNode::initialize_fields(GraphKit* kit, MultiNode* multi, uint& base_input, bool in, bool no_null_marker, Node* null_check_region, GrowableArray<ciType*>& visited) {
1429 PhaseGVN& gvn = kit->gvn();
1430 Node* null_marker = nullptr;
1431 if (!no_null_marker) {
1432 // Nullable inline type
1433 if (in) {
1434 // Set null marker
1435 if (multi->is_Start()) {
1436 null_marker = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1437 } else {
1438 null_marker = multi->as_Call()->in(base_input);
1439 }
1440 set_req(NullMarker, null_marker);
1441 base_input++;
1442 }
1443 // Add a null check to make subsequent loads dependent on
1444 assert(null_check_region == nullptr, "already set");
1445 if (null_marker == nullptr) {
1446 // Will only be initialized below, use dummy node for now
1447 null_marker = new Node(1);
1448 null_marker->init_req(0, kit->control()); // Add an input to prevent dummy from being dead
1449 gvn.set_type_bottom(null_marker);
1450 }
1451 Node* null_ctrl = kit->top();
1452 kit->null_check_common(null_marker, T_INT, false, &null_ctrl);
1453 Node* non_null_ctrl = kit->control();
1454 null_check_region = new RegionNode(3);
1455 null_check_region->init_req(1, non_null_ctrl);
1456 null_check_region->init_req(2, null_ctrl);
1457 null_check_region = gvn.transform(null_check_region);
1458 kit->set_control(null_check_region);
1459 }
1460
1461 for (uint i = 0; i < field_count(); ++i) {
1462 ciField* field = this->field(i);
1463 ciType* type = field->type();
1464 Node* parm = nullptr;
1465 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1466 if (field->is_flat()) {
1467 // Flat inline type field
1468 InlineTypeNode* vt = make_uninitialized(gvn, type->as_inline_klass(), field->is_null_free());
1469 vt->initialize_fields(kit, multi, base_input, in, true, null_check_region, visited);
1470 if (!field->is_null_free()) {
1471 assert(field->null_marker_offset() != -1, "inconsistency");
1472 Node* null_marker = nullptr;
1473 if (multi->is_Start()) {
1474 null_marker = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1475 } else if (in) {
1476 null_marker = multi->as_Call()->in(base_input);
1477 } else {
1478 null_marker = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1479 }
1480 vt->set_req(NullMarker, null_marker);
1481 base_input++;
1482 }
1483 parm = gvn.transform(vt);
1484 } else {
1485 if (multi->is_Start()) {
1486 assert(in, "return from start?");
1487 parm = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1488 } else if (in) {
1489 parm = multi->as_Call()->in(base_input);
1490 } else {
1491 parm = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1492 }
1493 // Non-flat inline type field
1494 if (type->is_inlinetype()) {
1495 if (null_check_region != nullptr) {
1496 // We limit scalarization for inline types with circular fields and can therefore observe nodes
1497 // of the same type but with different scalarization depth during GVN. To avoid inconsistencies
1498 // during merging, make sure that we only create Phis for fields that are guaranteed to be scalarized.
1499 if (parm->is_InlineType() && kit->C->has_circular_inline_type()) {
1500 parm = parm->as_InlineType()->get_oop();
1501 }
1502 // Holder is nullable, set field to nullptr if holder is nullptr to avoid loading from uninitialized memory
1503 parm = PhiNode::make(null_check_region, parm, TypeInstPtr::make(TypePtr::BotPTR, type->as_inline_klass()));
1504 parm->set_req(2, kit->zerocon(T_OBJECT));
1505 parm = gvn.transform(parm);
1506 }
1507 if (visited.contains(type)) {
1508 kit->C->set_has_circular_inline_type(true);
1509 } else if (!parm->is_InlineType()) {
1510 int old_len = visited.length();
1511 visited.push(type);
1512 parm = make_from_oop_impl(kit, parm, type->as_inline_klass(), visited);
1513 visited.trunc_to(old_len);
1514 }
1515 }
1516 base_input += type->size();
1517 }
1518 assert(parm != nullptr, "should never be null");
1519 assert(field_value(i) == nullptr, "already set");
1520 set_field_value(i, parm);
1521 gvn.record_for_igvn(parm);
1522 }
1523 // The last argument is used to pass the null marker to compiled code
1524 if (!no_null_marker && !in) {
1525 Node* cmp = null_marker->raw_out(0);
1526 null_marker = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1527 set_req(NullMarker, null_marker);
1528 gvn.hash_delete(cmp);
1529 cmp->set_req(1, null_marker);
1530 gvn.hash_find_insert(cmp);
1531 gvn.record_for_igvn(cmp);
1532 base_input++;
1533 }
1534 }
1535
1536 // Search for multiple allocations of this inline type and try to replace them by dominating allocations.
1537 // Equivalent InlineTypeNodes are merged by GVN, so we just need to search for AllocateNode users to find redundant allocations.
1538 void InlineTypeNode::remove_redundant_allocations(PhaseIdealLoop* phase) {
1539 PhaseIterGVN* igvn = &phase->igvn();
1540 // Search for allocations of this inline type. Ignore scalar replaceable ones, they
1541 // will be removed anyway and changing the memory chain will confuse other optimizations.
1542 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1543 AllocateNode* alloc = fast_out(i)->isa_Allocate();
1544 if (alloc != nullptr && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable) {
1545 Node* res = alloc->result_cast();
1546 if (res == nullptr || !res->is_CheckCastPP()) {
1547 break; // No unique CheckCastPP
1548 }
1549 // Search for a dominating allocation of the same inline type
1550 Node* res_dom = res;
1551 for (DUIterator_Fast jmax, j = fast_outs(jmax); j < jmax; j++) {
1552 AllocateNode* alloc_other = fast_out(j)->isa_Allocate();
1553 if (alloc_other != nullptr && alloc_other->in(AllocateNode::InlineType) == this && !alloc_other->_is_scalar_replaceable) {
1554 Node* res_other = alloc_other->result_cast();
1555 if (res_other != nullptr && res_other->is_CheckCastPP() && res_other != res_dom &&
1556 phase->is_dominator(res_other->in(0), res_dom->in(0))) {
1557 res_dom = res_other;
1558 }
1559 }
1560 }
1561 if (res_dom != res) {
1562 // Replace allocation by dominating one.
1563 replace_allocation(igvn, res, res_dom);
1564 // The result of the dominated allocation is now unused and will be removed
1565 // later in PhaseMacroExpand::eliminate_allocate_node to not confuse loop opts.
1566 igvn->_worklist.push(alloc);
1567 }
1568 }
1569 }
1570 }
1571
1572 InlineTypeNode* InlineTypeNode::make_null(PhaseGVN& gvn, ciInlineKlass* vk, bool transform) {
1573 GrowableArray<ciType*> visited;
1574 visited.push(vk);
1575 return make_null_impl(gvn, vk, visited, transform);
1576 }
1577
1578 InlineTypeNode* InlineTypeNode::make_null_impl(PhaseGVN& gvn, ciInlineKlass* vk, GrowableArray<ciType*>& visited, bool transform) {
1579 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT), /* null_free= */ false);
1580 vt->set_is_buffered(gvn);
1581 vt->set_null_marker(gvn, gvn.intcon(0));
1582 for (uint i = 0; i < vt->field_count(); i++) {
1583 ciField* field = vt->field(i);
1584 ciType* ft = field->type();
1585 Node* value = gvn.zerocon(ft->basic_type());
1586 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1587 if (!field->is_flat() && visited.contains(ft)) {
1588 gvn.C->set_has_circular_inline_type(true);
1589 } else if (ft->is_inlinetype()) {
1590 int old_len = visited.length();
1591 visited.push(ft);
1592 value = make_null_impl(gvn, ft->as_inline_klass(), visited);
1593 visited.trunc_to(old_len);
1594 }
1595 vt->set_field_value(i, value);
1596 }
1597 return transform ? gvn.transform(vt)->as_InlineType() : vt;
1598 }
1599
1600 InlineTypeNode* InlineTypeNode::clone_if_required(PhaseGVN* gvn, SafePointNode* map, bool safe_for_replace) {
1601 if (!safe_for_replace || (map == nullptr && outcnt() != 0)) {
1602 return clone()->as_InlineType();
1603 }
1604 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1605 if (fast_out(i) != map) {
1606 return clone()->as_InlineType();
1607 }
1608 }
1609 gvn->hash_delete(this);
1610 return this;
1611 }
1612
1613 const Type* InlineTypeNode::Value(PhaseGVN* phase) const {
1614 Node* oop = get_oop();
1615 const Type* toop = phase->type(oop);
1616 #ifdef ASSERT
1617 if (oop->is_Con() && toop->is_zero_type() && _type->isa_oopptr()->is_known_instance()) {
1618 // We are not allocated (anymore) and should therefore not have an instance id
1619 dump(1);
1620 assert(false, "Unbuffered inline type should not have known instance id");
1621 }
1622 #endif
1623 if (toop == Type::TOP) {
1624 return Type::TOP;
1625 }
1626 const Type* t = toop->filter_speculative(_type);
1627 if (t->singleton()) {
1628 // Don't replace InlineType by a constant
1629 t = _type;
1630 }
1631 const Type* tinit = phase->type(in(NullMarker));
1632 if (tinit == Type::TOP) {
1633 return Type::TOP;
1634 }
1635 if (tinit->isa_int() && tinit->is_int()->is_con(1)) {
1636 t = t->join_speculative(TypePtr::NOTNULL);
1637 }
1638 return t;
1639 }
1640
1641 InlineTypeNode* LoadFlatNode::load(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr, bool null_free, bool trust_null_free_oop, DecoratorSet decorators) {
1642 int output_type_size = vk->nof_nonstatic_fields() + (null_free ? 0 : 1);
1643 const Type** output_types = TypeTuple::fields(output_type_size);
1644 collect_field_types(vk, output_types + TypeFunc::Parms, 0, output_type_size, null_free, trust_null_free_oop);
1645 const TypeTuple* type = TypeTuple::make(output_type_size + TypeFunc::Parms, output_types);
1646
1647 LoadFlatNode* load = new LoadFlatNode(vk, type, null_free, decorators);
1648 load->init_req(TypeFunc::Control, kit->control());
1649 load->init_req(TypeFunc::I_O, kit->top());
1650 load->init_req(TypeFunc::Memory, kit->reset_memory());
1651 load->init_req(TypeFunc::FramePtr, kit->frameptr());
1652 load->init_req(TypeFunc::ReturnAdr, kit->top());
1653
1654 load->init_req(TypeFunc::Parms, base);
1655 load->init_req(TypeFunc::Parms + 1, ptr);
1656 kit->kill_dead_locals();
1657 kit->add_safepoint_edges(load);
1658 load = kit->gvn().transform(load)->as_LoadFlat();
1659 kit->record_for_igvn(load);
1660
1661 kit->set_control(kit->gvn().transform(new ProjNode(load, TypeFunc::Control)));
1662 kit->set_all_memory(kit->gvn().transform(new ProjNode(load, TypeFunc::Memory)));
1663 return load->collect_projs(kit, vk, TypeFunc::Parms, null_free);
1664 }
1665
1666 bool LoadFlatNode::expand_constant(PhaseIterGVN& igvn, ciInstance* inst) const {
1667 precond(inst != nullptr);
1668 assert(igvn.delay_transform(), "transformation must be delayed");
1669 if ((_decorators & C2_MISMATCHED) != 0) {
1670 return false;
1671 }
1672
1673 GraphKit kit(this, igvn);
1674 for (int i = 0; i < _vk->nof_nonstatic_fields(); i++) {
1675 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + i);
1676 if (proj_out == nullptr) {
1677 continue;
1678 }
1679
1680 ciField* field = _vk->nonstatic_field_at(i);
1681 BasicType bt = field->type()->basic_type();
1682 if (inst == nullptr) {
1683 Node* cst_node = igvn.zerocon(bt);
1684 igvn.replace_node(proj_out, cst_node);
1685 } else {
1686 bool is_unsigned_load = bt == T_BOOLEAN || bt == T_CHAR;
1687 const Type* cst_type = Type::make_constant_from_field(field, inst, bt, is_unsigned_load);
1688 Node* cst_node = igvn.makecon(cst_type);
1689 igvn.replace_node(proj_out, cst_node);
1690 }
1691 }
1692
1693 if (!_null_free) {
1694 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + _vk->nof_nonstatic_fields());
1695 if (proj_out != nullptr) {
1696 igvn.replace_node(proj_out, igvn.intcon(1));
1697 }
1698 }
1699
1700 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
1701 if (old_ctrl != nullptr) {
1702 igvn.replace_node(old_ctrl, kit.control());
1703 }
1704 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
1705 Node* new_mem = kit.reset_memory();
1706 if (old_mem != nullptr) {
1707 igvn.replace_node(old_mem, new_mem);
1708 }
1709 return true;
1710 }
1711
1712 bool LoadFlatNode::expand_non_atomic(PhaseIterGVN& igvn) {
1713 assert(igvn.delay_transform(), "transformation must be delayed");
1714 if ((_decorators & C2_MISMATCHED) != 0) {
1715 return false;
1716 }
1717
1718 GraphKit kit(this, igvn);
1719 Node* base = this->base();
1720 Node* ptr = this->ptr();
1721
1722 for (int i = 0; i < _vk->nof_nonstatic_fields(); i++) {
1723 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + i);
1724 if (proj_out == nullptr) {
1725 continue;
1726 }
1727
1728 ciField* field = _vk->nonstatic_field_at(i);
1729 Node* field_ptr = kit.basic_plus_adr(base, ptr, field->offset_in_bytes() - _vk->payload_offset());
1730 const TypePtr* field_ptr_type = field_ptr->Value(&igvn)->is_ptr();
1731 igvn.set_type(field_ptr, field_ptr_type);
1732
1733 Node* field_value = kit.access_load_at(base, field_ptr, field_ptr_type, igvn.type(proj_out), field->type()->basic_type(), _decorators);
1734 igvn.replace_node(proj_out, field_value);
1735 }
1736
1737 if (!_null_free) {
1738 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + _vk->nof_nonstatic_fields());
1739 if (proj_out != nullptr) {
1740 Node* null_marker_ptr = kit.basic_plus_adr(base, ptr, _vk->null_marker_offset_in_payload());
1741 const TypePtr* null_marker_ptr_type = null_marker_ptr->Value(&igvn)->is_ptr();
1742 igvn.set_type(null_marker_ptr, null_marker_ptr_type);
1743 Node* null_marker_value = kit.access_load_at(base, null_marker_ptr, null_marker_ptr_type, TypeInt::BOOL, T_BOOLEAN, _decorators);
1744 igvn.replace_node(proj_out, null_marker_value);
1745 }
1746 }
1747
1748 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
1749 if (old_ctrl != nullptr) {
1750 igvn.replace_node(old_ctrl, kit.control());
1751 }
1752 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
1753 Node* new_mem = kit.reset_memory();
1754 if (old_mem != nullptr) {
1755 igvn.replace_node(old_mem, new_mem);
1756 }
1757 return true;
1758 }
1759
1760 void LoadFlatNode::expand_atomic(PhaseIterGVN& igvn) {
1761 assert(igvn.delay_transform(), "transformation must be delayed");
1762 GraphKit kit(this, igvn);
1763 Node* base = this->base();
1764 Node* ptr = this->ptr();
1765
1766 BasicType payload_bt = _vk->atomic_size_to_basic_type(_null_free);
1767 kit.insert_mem_bar(Op_MemBarCPUOrder);
1768 Node* payload = kit.access_load_at(base, ptr, TypeRawPtr::BOTTOM, Type::get_const_basic_type(payload_bt), payload_bt,
1769 _decorators | C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD | C2_UNKNOWN_CONTROL_LOAD, kit.control());
1770 kit.insert_mem_bar(Op_MemBarCPUOrder);
1771
1772 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
1773 if (old_ctrl != nullptr) {
1774 igvn.replace_node(old_ctrl, kit.control());
1775 }
1776 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
1777 Node* new_mem = kit.reset_memory();
1778 if (old_mem != nullptr) {
1779 igvn.replace_node(old_mem, new_mem);
1780 }
1781
1782 expand_projs_atomic(igvn, kit.control(), payload);
1783 }
1784
1785 void LoadFlatNode::collect_field_types(ciInlineKlass* vk, const Type** field_types, int idx, int limit, bool null_free, bool trust_null_free_oop) {
1786 assert(null_free || !trust_null_free_oop, "cannot trust null-free oop when the holder object is not null-free");
1787 for (int i = 0; i < vk->nof_declared_nonstatic_fields(); i++) {
1788 ciField* field = vk->declared_nonstatic_field_at(i);
1789 if (field->is_flat()) {
1790 ciInlineKlass* field_klass = field->type()->as_inline_klass();
1791 collect_field_types(field_klass, field_types, idx, limit, field->is_null_free(), trust_null_free_oop && field->is_null_free());
1792 idx += field_klass->nof_nonstatic_fields() + (field->is_null_free() ? 0 : 1);
1793 continue;
1794 }
1795
1796 const Type* field_type = Type::get_const_type(field->type());
1797 if (trust_null_free_oop && field->is_null_free()) {
1798 field_type = field_type->filter(TypePtr::NOTNULL);
1799 }
1800
1801 assert(idx >= 0 && idx < limit, "field type out of bounds, %d - %d", idx, limit);
1802 field_types[idx] = field_type;
1803 idx++;
1804 }
1805
1806 if (!null_free) {
1807 assert(idx >= 0 && idx < limit, "field type out of bounds, %d - %d", idx, limit);
1808 field_types[idx] = TypeInt::BOOL;
1809 }
1810 }
1811
1812 // Create an InlineTypeNode from a LoadFlatNode with its fields being extracted from the
1813 // LoadFlatNode
1814 InlineTypeNode* LoadFlatNode::collect_projs(GraphKit* kit, ciInlineKlass* vk, int proj_con, bool null_free) {
1815 PhaseGVN& gvn = kit->gvn();
1816 InlineTypeNode* res = InlineTypeNode::make_uninitialized(gvn, vk, null_free);
1817 for (int i = 0; i < vk->nof_declared_nonstatic_fields(); i++) {
1818 ciField* field = vk->declared_nonstatic_field_at(i);
1819 Node* field_value;
1820 if (field->is_flat()) {
1821 ciInlineKlass* field_klass = field->type()->as_inline_klass();
1822 field_value = collect_projs(kit, field_klass, proj_con, field->is_null_free());
1823 proj_con += field_klass->nof_nonstatic_fields() + (field->is_null_free() ? 0 : 1);
1824 } else {
1825 field_value = gvn.transform(new ProjNode(this, proj_con));
1826 if (field->type()->is_inlinetype()) {
1827 field_value = InlineTypeNode::make_from_oop(kit, field_value, field->type()->as_inline_klass());
1828 }
1829 proj_con++;
1830 }
1831 res->set_field_value(i, field_value);
1832 }
1833
1834 if (null_free) {
1835 res->set_null_marker(gvn);
1836 } else {
1837 res->set_null_marker(gvn, gvn.transform(new ProjNode(this, proj_con)));
1838 }
1839 return gvn.transform(res)->as_InlineType();
1840 }
1841
1842 // Extract the values of the flattened fields from the loaded payload
1843 void LoadFlatNode::expand_projs_atomic(PhaseIterGVN& igvn, Node* ctrl, Node* payload) {
1844 BasicType payload_bt = _vk->atomic_size_to_basic_type(_null_free);
1845 for (int i = 0; i < _vk->nof_nonstatic_fields(); i++) {
1846 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + i);
1847 if (proj_out == nullptr) {
1848 continue;
1849 }
1850
1851 ciField* field = _vk->nonstatic_field_at(i);
1852 int field_offset = field->offset_in_bytes() - _vk->payload_offset();
1853 const Type* field_type = igvn.type(proj_out);
1854 Node* field_value = get_payload_value(igvn, ctrl, payload_bt, payload, field_type, field->type()->basic_type(), field_offset);
1855 igvn.replace_node(proj_out, field_value);
1856 }
1857
1858 if (!_null_free) {
1859 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + _vk->nof_nonstatic_fields());
1860 if (proj_out == nullptr) {
1861 return;
1862 }
1863
1864 int null_marker_offset = _vk->null_marker_offset_in_payload();
1865 Node* null_marker_value = get_payload_value(igvn, ctrl, payload_bt, payload, TypeInt::BOOL, T_BOOLEAN, null_marker_offset);
1866 igvn.replace_node(proj_out, null_marker_value);
1867 }
1868 }
1869
1870 Node* LoadFlatNode::get_payload_value(PhaseIterGVN& igvn, Node* ctrl, BasicType payload_bt, Node* payload, const Type* value_type, BasicType value_bt, int offset) {
1871 assert((offset + type2aelembytes(value_bt)) <= type2aelembytes(payload_bt), "Value does not fit into payload");
1872 Node* value = nullptr;
1873 // Shift to the right position in the long value
1874 Node* shift_val = igvn.intcon(offset << LogBitsPerByte);
1875 if (payload_bt == T_LONG) {
1876 value = igvn.transform(new URShiftLNode(payload, shift_val));
1877 value = igvn.transform(new ConvL2INode(value));
1878 } else {
1879 value = igvn.transform(new URShiftINode(payload, shift_val));
1880 }
1881
1882 if (value_bt == T_INT) {
1883 return value;
1884 } else if (!is_java_primitive(value_bt)) {
1885 assert(UseCompressedOops && payload_bt == T_LONG, "Naturally atomic");
1886 value = igvn.transform(new CastI2NNode(ctrl, value, value_type->make_narrowoop()));
1887 value = igvn.transform(new DecodeNNode(value, value_type));
1888
1889 // Similar to CheckCastPP nodes with raw input, CastI2N nodes require special handling in 'PhaseCFG::schedule_late' to ensure the
1890 // register allocator does not move the CastI2N below a safepoint. This is necessary to avoid having the raw pointer span a safepoint,
1891 // making it opaque to the GC. Unlike CheckCastPPs, which need extra handling in 'Scheduling::ComputeRegisterAntidependencies' due to
1892 // scalarization, CastI2N nodes are always used by a load if scalarization happens which inherently keeps them pinned above the safepoint.
1893 return value;
1894 } else {
1895 // Make sure to zero unused bits in the 32-bit value
1896 return Compile::narrow_value(value_bt, value, nullptr, &igvn, true);
1897 }
1898 }
1899
1900 void StoreFlatNode::store(GraphKit* kit, Node* base, Node* ptr, InlineTypeNode* value, bool null_free, DecoratorSet decorators) {
1901 value = value->allocate_fields(kit);
1902 StoreFlatNode* store = new StoreFlatNode(null_free, decorators);
1903 store->init_req(TypeFunc::Control, kit->control());
1904 store->init_req(TypeFunc::I_O, kit->top());
1905 store->init_req(TypeFunc::Memory, kit->reset_memory());
1906 store->init_req(TypeFunc::FramePtr, kit->frameptr());
1907 store->init_req(TypeFunc::ReturnAdr, kit->top());
1908
1909 store->init_req(TypeFunc::Parms, base);
1910 store->init_req(TypeFunc::Parms + 1, ptr);
1911 store->init_req(TypeFunc::Parms + 2, value);
1912 kit->kill_dead_locals();
1913 kit->add_safepoint_edges(store);
1914 store = kit->gvn().transform(store)->as_StoreFlat();
1915 kit->record_for_igvn(store);
1916
1917 kit->set_control(kit->gvn().transform(new ProjNode(store, TypeFunc::Control)));
1918 kit->set_all_memory(kit->gvn().transform(new ProjNode(store, TypeFunc::Memory)));
1919 }
1920
1921 bool StoreFlatNode::expand_non_atomic(PhaseIterGVN& igvn) {
1922 assert(igvn.delay_transform(), "transformation must be delayed");
1923 if ((_decorators & C2_MISMATCHED) != 0) {
1924 return false;
1925 }
1926
1927 GraphKit kit(this, igvn);
1928 Node* base = this->base();
1929 Node* ptr = this->ptr();
1930 InlineTypeNode* value = this->value();
1931
1932 ciInlineKlass* vk = igvn.type(value)->inline_klass();
1933 for (int i = 0; i < vk->nof_nonstatic_fields(); i++) {
1934 ciField* field = vk->nonstatic_field_at(i);
1935 Node* field_ptr = kit.basic_plus_adr(base, ptr, field->offset_in_bytes() - vk->payload_offset());
1936 const TypePtr* field_ptr_type = field_ptr->Value(&igvn)->is_ptr();
1937 igvn.set_type(field_ptr, field_ptr_type);
1938 Node* field_value = value->field_value_by_offset(field->offset_in_bytes(), true);
1939 Node* store = kit.access_store_at(base, field_ptr, field_ptr_type, field_value, igvn.type(field_value), field->type()->basic_type(), _decorators);
1940 }
1941
1942 if (!_null_free) {
1943 Node* null_marker_ptr = kit.basic_plus_adr(base, ptr, vk->null_marker_offset_in_payload());
1944 const TypePtr* null_marker_ptr_type = null_marker_ptr->Value(&igvn)->is_ptr();
1945 igvn.set_type(null_marker_ptr, null_marker_ptr_type);
1946 Node* null_marker_value = value->get_null_marker();
1947 Node* store = kit.access_store_at(base, null_marker_ptr, null_marker_ptr_type, null_marker_value, TypeInt::BOOL, T_BOOLEAN, _decorators);
1948 }
1949
1950 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
1951 if (old_ctrl != nullptr) {
1952 igvn.replace_node(old_ctrl, kit.control());
1953 }
1954 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
1955 Node* new_mem = kit.reset_memory();
1956 if (old_mem != nullptr) {
1957 igvn.replace_node(old_mem, new_mem);
1958 }
1959 return true;
1960 }
1961
1962 void StoreFlatNode::expand_atomic(PhaseIterGVN& igvn) {
1963 // Convert to a payload value <= 64-bit and write atomically.
1964 // The payload might contain at most two oop fields that must be narrow because otherwise they would be 64-bit
1965 // in size and would then be written by a "normal" oop store. If the payload contains oops, its size is always
1966 // 64-bit because the next smaller (power-of-two) size would be 32-bit which could only hold one narrow oop that
1967 // would then be written by a normal narrow oop store. These properties are asserted in 'convert_to_payload'.
1968 assert(igvn.delay_transform(), "transformation must be delayed");
1969 GraphKit kit(this, igvn);
1970 Node* base = this->base();
1971 Node* ptr = this->ptr();
1972 InlineTypeNode* value = this->value();
1973
1974 int oop_off_1 = -1;
1975 int oop_off_2 = -1;
1976 Node* payload = convert_to_payload(igvn, kit.control(), value, _null_free, oop_off_1, oop_off_2);
1977
1978 ciInlineKlass* vk = igvn.type(value)->inline_klass();
1979 assert(oop_off_1 == -1 || oop_off_1 == 0 || oop_off_1 == 4, "invalid layout for %s, first oop at offset %d", vk->name()->as_utf8(), oop_off_1);
1980 assert(oop_off_2 == -1 || oop_off_2 == 4, "invalid layout for %s, second oop at offset %d", vk->name()->as_utf8(), oop_off_2);
1981 BasicType payload_bt = vk->atomic_size_to_basic_type(_null_free);
1982 kit.insert_mem_bar(Op_MemBarCPUOrder);
1983 if (!UseG1GC || oop_off_1 == -1) {
1984 // No oop fields or no late barrier expansion. Emit an atomic store of the payload and add GC barriers if needed.
1985 assert(oop_off_2 == -1 || !UseG1GC, "sanity");
1986 // ZGC does not support compressed oops, so only one oop can be in the payload which is written by a "normal" oop store.
1987 assert((oop_off_1 == -1 && oop_off_2 == -1) || !UseZGC, "ZGC does not support embedded oops in flat fields");
1988 kit.access_store_at(base, ptr, TypeRawPtr::BOTTOM, payload, Type::get_const_basic_type(payload_bt), payload_bt, _decorators | C2_MISMATCHED, true, value);
1989 } else {
1990 // Contains oops and requires late barrier expansion. Emit a special store node that allows to emit GC barriers in the backend.
1991 assert(UseG1GC, "Unexpected GC");
1992 assert(payload_bt == T_LONG, "Unexpected payload type");
1993 // If one oop, set the offset (if no offset is set, two oops are assumed by the backend)
1994 Node* oop_offset = (oop_off_2 == -1) ? igvn.intcon(oop_off_1) : nullptr;
1995 Node* mem = kit.reset_memory();
1996 kit.set_all_memory(mem);
1997 Node* store = igvn.transform(new StoreLSpecialNode(kit.control(), mem, ptr, TypeRawPtr::BOTTOM, payload, oop_offset, MemNode::unordered));
1998 kit.set_memory(store, TypeRawPtr::BOTTOM);
1999 }
2000 kit.insert_mem_bar(Op_MemBarCPUOrder);
2001
2002 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
2003 if (old_ctrl != nullptr) {
2004 igvn.replace_node(old_ctrl, kit.control());
2005 }
2006 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
2007 Node* new_mem = kit.reset_memory();
2008 if (old_mem != nullptr) {
2009 igvn.replace_node(old_mem, new_mem);
2010 }
2011 }
2012
2013 // Convert the field values to a payload value of type 'bt'
2014 Node* StoreFlatNode::convert_to_payload(PhaseIterGVN& igvn, Node* ctrl, InlineTypeNode* value, bool null_free, int& oop_off_1, int& oop_off_2) {
2015 ciInlineKlass* vk = igvn.type(value)->inline_klass();
2016 BasicType payload_bt = vk->atomic_size_to_basic_type(null_free);
2017 Node* payload = igvn.zerocon(payload_bt);
2018 if (!null_free) {
2019 // Set the null marker
2020 payload = set_payload_value(igvn, payload_bt, payload, T_BOOLEAN, value->get_null_marker(), vk->null_marker_offset_in_payload());
2021 }
2022
2023 // Iterate over the fields and add their values to the payload
2024 for (int i = 0; i < vk->nof_nonstatic_fields(); i++) {
2025 ciField* field = vk->nonstatic_field_at(i);
2026 Node* field_value = value->field_value_by_offset(field->offset_in_bytes(), true);
2027 ciType* field_klass = field->type();
2028 BasicType field_bt = field_klass->basic_type();
2029 int field_offset_in_payload = field->offset_in_bytes() - vk->payload_offset();
2030 if (!field_klass->is_primitive_type()) {
2031 // Narrow oop field
2032 assert(UseCompressedOops && payload_bt == T_LONG, "Naturally atomic");
2033 if (oop_off_1 == -1) {
2034 oop_off_1 = field_offset_in_payload;
2035 } else {
2036 assert(oop_off_2 == -1, "already set");
2037 oop_off_2 = field_offset_in_payload;
2038 }
2039
2040 const Type* val_type = Type::get_const_type(field_klass)->make_narrowoop();
2041 if (field_value->is_InlineType()) {
2042 assert(field_value->as_InlineType()->is_allocated(&igvn), "must be allocated");
2043 }
2044
2045 field_value = igvn.transform(new EncodePNode(field_value, val_type));
2046 field_value = igvn.transform(new CastP2XNode(ctrl, field_value));
2047 field_value = igvn.transform(new ConvL2INode(field_value));
2048 field_bt = T_INT;
2049 }
2050 payload = set_payload_value(igvn, payload_bt, payload, field_bt, field_value, field_offset_in_payload);
2051 }
2052
2053 return payload;
2054 }
2055
2056 Node* StoreFlatNode::set_payload_value(PhaseIterGVN& igvn, BasicType payload_bt, Node* payload, BasicType val_bt, Node* value, int offset) {
2057 assert((offset + type2aelembytes(val_bt)) <= type2aelembytes(payload_bt), "Value does not fit into payload");
2058
2059 // Make sure to zero unused bits in the 32-bit value
2060 if (val_bt == T_BYTE || val_bt == T_BOOLEAN) {
2061 value = igvn.transform(new AndINode(value, igvn.intcon(0xFF)));
2062 } else if (val_bt == T_CHAR || val_bt == T_SHORT) {
2063 value = igvn.transform(new AndINode(value, igvn.intcon(0xFFFF)));
2064 } else if (val_bt == T_FLOAT) {
2065 value = igvn.transform(new MoveF2INode(value));
2066 } else {
2067 assert(val_bt == T_INT, "Unsupported type: %s", type2name(val_bt));
2068 }
2069
2070 Node* shift_val = igvn.intcon(offset << LogBitsPerByte);
2071 if (payload_bt == T_LONG) {
2072 // Convert to long and remove the sign bit (the backend will fold this and emit a zero extend i2l)
2073 value = igvn.transform(new ConvI2LNode(value));
2074 value = igvn.transform(new AndLNode(value, igvn.longcon(0xFFFFFFFF)));
2075
2076 Node* shift_value = igvn.transform(new LShiftLNode(value, shift_val));
2077 payload = new OrLNode(shift_value, payload);
2078 } else {
2079 Node* shift_value = igvn.transform(new LShiftINode(value, shift_val));
2080 payload = new OrINode(shift_value, payload);
2081 }
2082 return igvn.transform(payload);
2083 }
2084
2085 const Type* LoadFlatNode::Value(PhaseGVN* phase) const {
2086 if (phase->type(in(TypeFunc::Control)) == Type::TOP || phase->type(in(TypeFunc::Memory)) == Type::TOP ||
2087 phase->type(base()) == Type::TOP || phase->type(ptr()) == Type::TOP) {
2088 return Type::TOP;
2089 }
2090 return bottom_type();
2091 }
2092
2093 const Type* StoreFlatNode::Value(PhaseGVN* phase) const {
2094 if (phase->type(in(TypeFunc::Control)) == Type::TOP || phase->type(in(TypeFunc::Memory)) == Type::TOP ||
2095 phase->type(base()) == Type::TOP || phase->type(ptr()) == Type::TOP || phase->type(value()) == Type::TOP) {
2096 return Type::TOP;
2097 }
2098 return bottom_type();
2099 }